You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2017/12/12 23:58:36 UTC

[01/50] hadoop git commit: HDFS-12889. Addendum patch to add missing file.

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 4541ead5e -> 070ad8438


HDFS-12889. Addendum patch to add missing file.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cca5b3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cca5b3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cca5b3b

Branch: refs/heads/HDFS-7240
Commit: 6cca5b3bcb440095f12d3eda88101fa250ac000a
Parents: 40b0045e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Dec 7 13:22:48 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Dec 7 13:23:37 2017 +0900

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cca5b3b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
new file mode 100644
index 0000000..77470cb
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[03/50] hadoop git commit: MAPREDUCE-6998. Moving logging APIs over to slf4j in hadoop-mapreduce-client-jobclient. Contributed by Gergely Novák.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java
index 3dbc577..9a5ca07 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapred;
 import java.io.DataOutputStream;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -34,13 +32,15 @@ import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A JUnit test to test Job System Directory with Mini-DFS.
  */
 public class TestJobSysDirWithDFS {
-  private static final Log LOG =
-    LogFactory.getLog(TestJobSysDirWithDFS.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestJobSysDirWithDFS.class);
   
   static final int NUM_MAPS = 10;
   static final int NUM_SAMPLES = 100000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
index bacc196..0991ae0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestKeyValueTextInputFormat.java
@@ -21,19 +21,21 @@ package org.apache.hadoop.mapred;
 import java.io.*;
 import java.util.*;
 
-import org.apache.commons.logging.*;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.compress.*;
 import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 
 public class TestKeyValueTextInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestKeyValueTextInputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestKeyValueTextInputFormat.class);
 
   private static int MAX_LENGTH = 10000;
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 19313d3..a23caa5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -31,8 +31,6 @@ import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -59,14 +57,16 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Sets;
 
 public class TestMRTimelineEventHandling {
 
   private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
-  private static final Log LOG =
-      LogFactory.getLog(TestMRTimelineEventHandling.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRTimelineEventHandling.class);
 
   @Test
   public void testTimelineServiceStartInMiniCluster() throws Exception {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
index b8ff016..f364c18 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java
@@ -22,8 +22,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.ipc.ProtocolSignature;
@@ -39,6 +37,9 @@ import org.apache.hadoop.mapreduce.split.JobSplitWriter;
 import org.apache.hadoop.mapreduce.split.SplitMetaInfoReader;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -56,7 +57,8 @@ import static org.junit.Assert.assertTrue;
  *  validated here.
  */
 public class TestMapProgress {
-  public static final Log LOG = LogFactory.getLog(TestMapProgress.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestMapProgress.class);
   private static String TEST_ROOT_DIR;
   static {
     String root = new File(System.getProperty("test.build.data", "/tmp"))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
index 51f0120..340cba8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
@@ -27,8 +27,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -46,6 +44,8 @@ import org.apache.hadoop.util.Shell;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Class to test mapred task's 
@@ -53,8 +53,8 @@ import org.junit.Test;
  *   - child env
  */
 public class TestMiniMRChildTask {
-  private static final Log LOG =
-    LogFactory.getLog(TestMiniMRChildTask.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMiniMRChildTask.class);
 
   private final static String OLD_CONFIGS = "test.old.configs";
   private final static String TASK_OPTS_VAL = "-Xmx200m";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
index 1bd2954..3a02f4c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileInputFormat.java
@@ -21,13 +21,14 @@ import java.io.IOException;
 import java.util.BitSet;
 import java.util.HashMap;
 import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 
@@ -35,7 +36,8 @@ public class TestMultiFileInputFormat {
 
   private static JobConf job = new JobConf();
 
-  private static final Log LOG = LogFactory.getLog(TestMultiFileInputFormat.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMultiFileInputFormat.class);
   
   private static final int MAX_SPLIT_COUNT  = 10000;
   private static final int SPLIT_COUNT_INCR = 6000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
index 03dc6a6..ec53892 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSequenceFileAsBinaryOutputFormat.java
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.io.IOException;
+import java.util.Random;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.BooleanWritable;
@@ -31,17 +31,16 @@ import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Random;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TestSequenceFileAsBinaryOutputFormat {
-  private static final Log LOG =
-      LogFactory.getLog(TestSequenceFileAsBinaryOutputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSequenceFileAsBinaryOutputFormat.class);
   private static final int RECORDS = 10000;
   // A random task attempt id for testing.
   private static final String attempt = "attempt_200707121733_0001_m_000000_0";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
index 82d1d2d..ffa42e4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSortedRanges.java
@@ -17,18 +17,18 @@
  */
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.SortedRanges.Range;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
 
 public class TestSortedRanges {
-  private static final Log LOG =
-    LogFactory.getLog(TestSortedRanges.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSortedRanges.class);
 
   @Test
   public void testAdd() {
@@ -56,7 +56,7 @@ public class TestSortedRanges {
     sr.add(new Range(21,50));
     assertEquals(70, sr.getIndicesCount());
     
-    LOG.debug(sr);
+    LOG.debug(sr.toString());
     
     Iterator<Long> it = sr.skipRangeIterator();
     int i = 0;
@@ -96,7 +96,7 @@ public class TestSortedRanges {
     sr.remove(new SortedRanges.Range(5,1));
     assertEquals(3, sr.getIndicesCount());
     
-    LOG.debug(sr);
+    LOG.debug(sr.toString());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
index b9e3275..5b4a141 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java
@@ -18,8 +18,9 @@
 
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URI;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -30,10 +31,8 @@ import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
 import org.apache.hadoop.util.Progressable;
 import org.junit.Test;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.net.URI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -42,8 +41,8 @@ import static org.junit.Assert.fail;
  * A JUnit test to test that jobs' output filenames are not HTML-encoded (cf HADOOP-1795).
  */
 public class TestSpecialCharactersInOutputPath {
-  private static final Log LOG =
-    LogFactory.getLog(TestSpecialCharactersInOutputPath.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSpecialCharactersInOutputPath.class);
   
   private static final String OUTPUT_FILENAME = "result[0]";
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
index e71103d..94299b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTaskStatus.java
@@ -17,13 +17,11 @@
  */
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.junit.Test;
+
 import static org.junit.Assert.*;
 
 public class TestTaskStatus {
-  private static final Log LOG = LogFactory.getLog(TestTaskStatus.class);
 
   @Test
   public void testMapTaskStatusStartAndFinishTimes() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
index 11f0bb5..0ea1d6d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java
@@ -38,16 +38,15 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.*;
 import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ReflectionUtils;
-
 import org.junit.Test;
-import static org.junit.Assert.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.*;
 
 public class TestTextInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestTextInputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestTextInputFormat.class);
 
   private static int MAX_LENGTH = 10000;
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
index 378363b..10d4c21 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java
@@ -48,8 +48,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -117,7 +115,6 @@ import org.apache.log4j.Appender;
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Layout;
 import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
 import org.apache.log4j.SimpleLayout;
 import org.apache.log4j.WriterAppender;
 import org.apache.log4j.spi.LoggingEvent;
@@ -128,6 +125,8 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableList;
 
@@ -136,7 +135,8 @@ import com.google.common.collect.ImmutableList;
  * fine
  */
 public class TestYARNRunner {
-  private static final Log LOG = LogFactory.getLog(TestYARNRunner.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestYARNRunner.class);
   private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
 
   // prefix before <LOG_DIR>/profile.out
@@ -574,7 +574,8 @@ public class TestYARNRunner {
   }
   @Test(timeout=20000)
   public void testWarnCommandOpts() throws Exception {
-    Logger logger = Logger.getLogger(YARNRunner.class);
+    org.apache.log4j.Logger logger =
+        org.apache.log4j.Logger.getLogger(YARNRunner.class);
     
     ByteArrayOutputStream bout = new ByteArrayOutputStream();
     Layout layout = new SimpleLayout();
@@ -1006,7 +1007,8 @@ public class TestYARNRunner {
         MRJobConfig.RESOURCE_TYPE_NAME_MEMORY,
         MRJobConfig.RESOURCE_TYPE_ALTERNATIVE_NAME_MEMORY)) {
       TestAppender testAppender = new TestAppender();
-      Logger logger = Logger.getLogger(YARNRunner.class);
+      org.apache.log4j.Logger  logger =
+          org.apache.log4j.Logger.getLogger(YARNRunner.class);
       logger.addAppender(testAppender);
       try {
         JobConf jobConf = new JobConf();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
index c607bfb..68fe1a0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ThreadedMapBenchmark.java
@@ -22,8 +22,6 @@ import java.io.IOException;
 import java.io.File;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -36,6 +34,8 @@ import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Distributed threaded map benchmark.
@@ -52,7 +52,8 @@ import org.apache.hadoop.util.ToolRunner;
 
 public class ThreadedMapBenchmark extends Configured implements Tool {
 
-  private static final Log LOG = LogFactory.getLog(ThreadedMapBenchmark.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ThreadedMapBenchmark.class);
   private static Path BASE_DIR =
     new Path(System.getProperty("test.build.data", 
                                 File.separator + "benchmarks" + File.separator 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index 935c175..89ac3ea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -34,8 +34,6 @@ import java.util.Properties;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeoutException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -60,6 +58,8 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Supplier;
 
@@ -69,7 +69,7 @@ import com.google.common.base.Supplier;
  */
 public class UtilsForTests {
 
-  static final Log LOG = LogFactory.getLog(UtilsForTests.class);
+  static final Logger LOG = LoggerFactory.getLogger(UtilsForTests.class);
 
   final static long KB = 1024L * 1;
   final static long MB = 1024L * KB;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
index 07b1306..3cd9c24 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/jobcontrol/TestLocalJobControl.java
@@ -21,13 +21,14 @@ package org.apache.hadoop.mapred.jobcontrol;
 import java.io.IOException;
 import java.util.ArrayList;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.mapred.JobConf;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -35,8 +36,8 @@ import static org.junit.Assert.assertEquals;
  */
 public class TestLocalJobControl extends HadoopTestCase {
 
-  public static final Log LOG = LogFactory.getLog(TestLocalJobControl.class
-      .getName());
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestLocalJobControl.class);
 
   /**
    * Initialises a new instance of this test case to use a Local MR cluster and

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java
index 84b491a..f7ef958 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java
@@ -23,8 +23,6 @@ import java.io.IOException;
 import java.util.List;
 import java.util.ArrayList;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.fs.FileUtil;
@@ -45,14 +43,16 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
 
 @Ignore
 public class TestPipes {
-  private static final Log LOG =
-    LogFactory.getLog(TestPipes.class.getName());
+  private static final Logger LOG = LoggerFactory.getLogger(TestPipes.class);
 
   private static Path cppExamples =
     new Path(System.getProperty("install.c++.examples"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java
index 55740f7..92b72cc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/GrowingSleepJob.java
@@ -17,20 +17,20 @@
  */
 package org.apache.hadoop.mapreduce;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.io.IOException;
+import java.util.ArrayList;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-import java.util.ArrayList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A sleep job whose mappers create 1MB buffer for every record.
  */
 public class GrowingSleepJob extends SleepJob {
-  private static final Log LOG = LogFactory.getLog(GrowingSleepJob.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(GrowingSleepJob.class);
 
   public static class GrowingSleepMapper extends SleepMapper {
     private final int MB = 1024 * 1024;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
index 5d9dc0b..3783466 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
@@ -20,19 +20,20 @@ package org.apache.hadoop.mapreduce;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Used to parse job history and configuration files.
  */
 class JobHistoryFileParser {
-  private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JobHistoryFileParser.class);
 
   private final FileSystem fs;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayHelper.java
index 8acd26e..4b47b43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayHelper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayHelper.java
@@ -24,19 +24,18 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class JobHistoryFileReplayHelper {
-  private static final Log LOG =
-      LogFactory.getLog(JobHistoryFileReplayHelper.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JobHistoryFileReplayHelper.class);
   static final String PROCESSING_PATH = "processing path";
   static final String REPLAY_MODE = "replay mode";
   static final int WRITE_ALL_AT_ONCE = 1;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
index d553596..1d0896f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
@@ -23,8 +23,6 @@ import java.util.Collection;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
@@ -38,7 +36,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.exceptions.YarnException;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Mapper for TimelineServicePerformanceV1 that replays job history files to the
@@ -48,8 +47,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 class JobHistoryFileReplayMapperV1 extends
     org.apache.hadoop.mapreduce.
         Mapper<IntWritable,IntWritable,Writable,Writable> {
-  private static final Log LOG =
-      LogFactory.getLog(JobHistoryFileReplayMapperV1.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JobHistoryFileReplayMapperV1.class);
 
   public void map(IntWritable key, IntWritable val, Context context) throws IOException {
     // collect the apps it needs to process

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
index 2ec4833..f8e6ca0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
@@ -23,8 +23,6 @@ import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.mapreduce.JobHistoryFileReplayHelper.JobFiles;
@@ -38,6 +36,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Mapper for TimelineServicePerformance that replays job history files to the
@@ -45,8 +45,8 @@ import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector
  *
  */
 class JobHistoryFileReplayMapperV2 extends EntityWriterV2 {
-  private static final Log LOG =
-      LogFactory.getLog(JobHistoryFileReplayMapperV2.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JobHistoryFileReplayMapperV2.class);
 
   @Override
   protected void writeEntities(Configuration tlConf,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MapReduceTestUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
index 8351b53..2f30bb5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
@@ -31,8 +31,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -46,22 +44,19 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.TaskLog;
 import org.apache.hadoop.mapred.Utils;
-import org.apache.hadoop.mapred.TaskLog.LogName;
-import org.apache.hadoop.mapred.TaskLog.Reader;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Utility methods used in various Job Control unit tests.
  */
 public class MapReduceTestUtil {
-  public static final Log LOG = 
-    LogFactory.getLog(MapReduceTestUtil.class.getName());
+  public static final Logger LOG =
+      LoggerFactory.getLogger(MapReduceTestUtil.class);
 
   static private Random rand = new Random();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
index 324f0ca..b89cdc0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java
@@ -34,8 +34,6 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -46,6 +44,8 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.eclipse.jetty.util.ajax.JSON;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class drives the creation of a mini-cluster on the local machine. By
@@ -60,8 +60,8 @@ import org.eclipse.jetty.util.ajax.JSON;
  * To shutdown the cluster, kill the process.
  */
 public class MiniHadoopClusterManager {
-  private static final Log LOG = LogFactory
-      .getLog(MiniHadoopClusterManager.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MiniHadoopClusterManager.class);
 
   private MiniMRClientCluster mr;
   private MiniDFSCluster dfs;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
index 6d6151f..219f552 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
@@ -22,8 +22,6 @@ import java.io.IOException;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Writable;
@@ -32,6 +30,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
    * Adds simple entities with random string payload, events, metrics, and
@@ -41,7 +41,8 @@ class SimpleEntityWriterV1
     extends org.apache.hadoop.mapreduce.Mapper
         <IntWritable, IntWritable, Writable, Writable>
     implements SimpleEntityWriterConstants {
-  private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV1.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SimpleEntityWriterV1.class);
 
   public void map(IntWritable key, IntWritable val, Context context)
       throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
index d66deb0..93633a06 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
@@ -22,8 +22,6 @@ import java.io.IOException;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -35,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
 import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Adds simple entities with random string payload, events, metrics, and
@@ -42,7 +42,8 @@ import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector
  */
 class SimpleEntityWriterV2 extends EntityWriterV2
     implements SimpleEntityWriterConstants {
-  private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV2.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SimpleEntityWriterV2.class);
 
   protected void writeEntities(Configuration tlConf,
       TimelineCollectorManager manager, Context context) throws IOException {
@@ -128,4 +129,4 @@ class SimpleEntityWriterV2 extends EntityWriterV2
       manager.remove(appId);
     }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
index 83d689c..8c79435 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
@@ -19,20 +19,19 @@ package org.apache.hadoop.mapreduce;
 
 import java.util.Random;
 
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.counters.LimitExceededException;
 import org.apache.hadoop.mapreduce.counters.Limits;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.Test;
 
+import static org.junit.Assert.*;
 /**
  * TestCounters checks the sanity and recoverability of {@code Counters}
  */
 public class TestCounters {
 
-  static final Log LOG = LogFactory.getLog(TestCounters.class);
+  static final Logger LOG = LoggerFactory.getLogger(TestCounters.class);
 
   /**
    * Verify counter value works

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java
index 8177ecd..7630e85 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestLocalRunner.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.mapreduce;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -33,6 +31,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -54,7 +54,8 @@ import static org.junit.Assert.assertTrue;
  */
 public class TestLocalRunner {
 
-  private static final Log LOG = LogFactory.getLog(TestLocalRunner.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestLocalRunner.class);
 
   private static int INPUT_SIZES[] =
     new int[] { 50000, 500, 500, 20,  5000, 500};
@@ -330,9 +331,9 @@ public class TestLocalRunner {
     try {
       job.waitForCompletion(true);
     } catch (InterruptedException ie) {
-      LOG.fatal("Interrupted while waiting for job completion", ie);
+      LOG.error("Interrupted while waiting for job completion", ie);
       for (int i = 0; i < 10; i++) {
-        LOG.fatal("Dumping stacks");
+        LOG.error("Dumping stacks");
         ReflectionUtils.logThreadInfo(LOG, "multimap threads", 0);
         Thread.sleep(1000);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 3a33bb1..f4ccc56 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.mapreduce;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;
@@ -33,6 +31,8 @@ import org.apache.hadoop.util.ToolRunner;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
@@ -60,7 +60,8 @@ import static org.junit.Assert.fail;
  */
 public class TestMRJobClient extends ClusterMapReduceTestCase {
 
-  private static final Log LOG = LogFactory.getLog(TestMRJobClient.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRJobClient.class);
 
   private Job runJob(Configuration conf) throws Exception {
     String input = "hello1\nhello2\nhello3\n";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java
index afe4a10..b6ec120 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java
@@ -24,22 +24,21 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
-import org.junit.Test;
-import static org.junit.Assert.*;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
 
 public class TestMapCollection {
 
-  private static final Log LOG = LogFactory.getLog(
+  private static final Logger LOG = LoggerFactory.getLogger(
       TestMapCollection.class.getName());
 
   public static abstract class FillWritable implements Writable, Configurable {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
index b757fb2..4bcacd8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestValueIterReset.java
@@ -27,8 +27,6 @@ import java.io.Writer;
 import java.util.ArrayList;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
@@ -42,6 +40,8 @@ import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertTrue;
 
@@ -66,8 +66,8 @@ public class TestValueIterReset {
     }
   }
 
-  private static final Log LOG =
-    LogFactory.getLog(TestValueIterReset.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestValueIterReset.class);
 
   public static class TestMapper 
   extends Mapper<LongWritable, Text, IntWritable, IntWritable> {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
index dcc3ce0..a6fc58e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
@@ -22,17 +22,17 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class TimelineEntityConverterV1 {
-  private static final Log LOG =
-      LogFactory.getLog(TimelineEntityConverterV1.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TimelineEntityConverterV1.class);
 
   static final String JOB = "MAPREDUCE_JOB";
   static final String TASK = "MAPREDUCE_TASK";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
index 45812fe..48c470b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
@@ -24,8 +24,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
 import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
@@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 
 class TimelineEntityConverterV2 {
-  private static final Log LOG =
-      LogFactory.getLog(TimelineEntityConverterV2.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TimelineEntityConverterV2.class);
 
   static final String JOB = "MAPREDUCE_JOB";
   static final String TASK = "MAPREDUCE_TASK";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDataDrivenDBInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDataDrivenDBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDataDrivenDBInputFormat.java
index 81a3249..39b63d5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDataDrivenDBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDataDrivenDBInputFormat.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.mapreduce.lib.db;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -36,6 +34,8 @@ import org.hsqldb.server.Server;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -58,7 +58,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class TestDataDrivenDBInputFormat extends HadoopTestCase {
 
-  private static final Log LOG = LogFactory.getLog(
+  private static final Logger LOG = LoggerFactory.getLogger(
       TestDataDrivenDBInputFormat.class);
 
   private static final String DB_NAME = "dddbif";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineSequenceFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineSequenceFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineSequenceFileInputFormat.java
index cb54158..dbbdc3d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineSequenceFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineSequenceFileInputFormat.java
@@ -27,8 +27,6 @@ import java.util.BitSet;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -44,10 +42,12 @@ import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestCombineSequenceFileInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestCombineSequenceFileInputFormat.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCombineSequenceFileInputFormat.class);
   private static Configuration conf = new Configuration();
   private static FileSystem localFs = null;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineTextInputFormat.java
index cb1dffc..3dd2333 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineTextInputFormat.java
@@ -32,8 +32,6 @@ import java.util.BitSet;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -51,10 +49,12 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestCombineTextInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestCombineTextInputFormat.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCombineTextInputFormat.class);
 
   private static Configuration defaultConf = new Configuration();
   private static FileSystem localFs = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
index b82b495..684d3e1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java
@@ -26,8 +26,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -41,16 +39,19 @@ import org.apache.hadoop.mapreduce.MapReduceTestUtil;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
-
 import org.apache.hadoop.util.ReflectionUtils;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.*;
 
 public class TestFixedLengthInputFormat {
 
-  private static Log LOG;
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFixedLengthInputFormat.class);
+
   private static Configuration defaultConf;
   private static FileSystem localFs;
   private static Path workDir;
@@ -62,7 +63,6 @@ public class TestFixedLengthInputFormat {
   @BeforeClass
   public static void onlyOnce() {
     try {
-      LOG = LogFactory.getLog(TestFixedLengthInputFormat.class.getName());
       defaultConf = new Configuration();
       defaultConf.set("fs.defaultFS", "file:///");
       localFs = FileSystem.getLocal(defaultConf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java
index f181c9c..537d23c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java
@@ -28,7 +28,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.*;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.io.*;
@@ -42,16 +41,15 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.apache.hadoop.util.LineReader;
 import org.apache.hadoop.util.ReflectionUtils;
-
 import org.junit.Test;
-import static org.junit.Assert.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.*;
 
 public class TestMRKeyValueTextInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestMRKeyValueTextInputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRKeyValueTextInputFormat.class);
 
   private static Configuration defaultConf = new Configuration();
   private static FileSystem localFs = null; 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java
index 89aa7b2..178eb50 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRSequenceFileInputFilter.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.mapreduce.lib.input;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -35,6 +33,8 @@ import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Random;
@@ -42,8 +42,8 @@ import java.util.Random;
 import static org.junit.Assert.assertEquals;
 
 public class TestMRSequenceFileInputFilter {
-  private static final Log LOG =
-    LogFactory.getLog(TestMRSequenceFileInputFilter.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRSequenceFileInputFilter.class);
 
   private static final int MAX_LENGTH = 15000;
   private static final Configuration conf = new Configuration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
index da011a2..534593e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControl.java
@@ -22,17 +22,17 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.junit.Assert;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MapReduceTestUtil;
+import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -43,8 +43,8 @@ import static org.junit.Assert.assertFalse;
  */
 public class TestMapReduceJobControl extends HadoopTestCase {
 
-  public static final Log LOG = 
-      LogFactory.getLog(TestMapReduceJobControl.class.getName());
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestMapReduceJobControl.class);
 
   static Path rootDataDir = new Path(
     System.getProperty("test.build.data", "."), "TestData");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java
index 5a8aeda..ad213d3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestMRSequenceFileAsBinaryOutputFormat.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.mapreduce.lib.output;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -44,6 +42,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.Random;
@@ -53,8 +53,8 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TestMRSequenceFileAsBinaryOutputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestMRSequenceFileAsBinaryOutputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRSequenceFileAsBinaryOutputFormat.class);
 
   private static final int RECORDS = 10000;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java
index 4d05d13..54c4cc3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestKeyFieldHelper.java
@@ -17,15 +17,17 @@
  */
 package org.apache.hadoop.mapreduce.lib.partition;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 public class TestKeyFieldHelper {
-  private static final Log LOG = LogFactory.getLog(TestKeyFieldHelper.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestKeyFieldHelper.class);
+
   /**
    * Test is key-field-helper's parse option.
    */
@@ -427,4 +429,4 @@ public class TestKeyFieldHelper {
     }
     return true;
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
index 387e202..d8c4b2a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java
@@ -28,8 +28,6 @@ import java.security.PrivilegedExceptionAction;
 
 import org.junit.Assert;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.mapred.JobConf;
@@ -54,17 +52,19 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestJHSSecurity {
 
-  private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestJHSSecurity.class);
   
   @Test
   public void testDelegationToken() throws IOException, InterruptedException {
 
-    Logger rootLogger = LogManager.getRootLogger();
+    org.apache.log4j.Logger rootLogger = LogManager.getRootLogger();
     rootLogger.setLevel(Level.DEBUG);
 
     final YarnConfiguration conf = new YarnConfiguration(new JobConf());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
index be46385..0c7a173 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java
@@ -24,8 +24,6 @@ import java.util.Date;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -34,6 +32,8 @@ import org.apache.hadoop.util.AsyncDiskService;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is a container of multiple thread pools, each for a volume,
@@ -53,7 +53,8 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceStability.Unstable
 public class MRAsyncDiskService {
   
-  public static final Log LOG = LogFactory.getLog(MRAsyncDiskService.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(MRAsyncDiskService.class);
   
   AsyncDiskService asyncDiskService;
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java
index 07b5d8b..fbccd9f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestMRAsyncDiskService.java
@@ -20,16 +20,14 @@ package org.apache.hadoop.mapreduce.util;
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.mapreduce.util.MRAsyncDiskService;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -42,7 +40,8 @@ import static org.junit.Assert.fail;
  */
 public class TestMRAsyncDiskService {
 
-  public static final Log LOG = LogFactory.getLog(TestMRAsyncDiskService.class);
+  public static final Logger LOG =
+      LoggerFactory.getLogger(TestMRAsyncDiskService.class);
   
   private static String TEST_ROOT_DIR = new Path(System.getProperty(
       "test.build.data", "/tmp")).toString();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[05/50] hadoop git commit: MAPREDUCE-7000. Moving logging APIs over to slf4j in hadoop-mapreduce-client-nativetask. Contributed by Jinjiang Ling.

Posted by xy...@apache.org.
MAPREDUCE-7000. Moving logging APIs over to slf4j in hadoop-mapreduce-client-nativetask. Contributed by Jinjiang Ling.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2e8a5c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2e8a5c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2e8a5c2

Branch: refs/heads/HDFS-7240
Commit: c2e8a5c2296a1e5033f5750d9d0a0c668ec7162f
Parents: d4cae97
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Dec 7 16:27:08 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Dec 7 16:27:08 2017 +0900

----------------------------------------------------------------------
 .../org/apache/hadoop/mapred/nativetask/HadoopPlatform.java | 7 ++++---
 .../hadoop/mapred/nativetask/NativeBatchProcessor.java      | 9 +++++----
 .../nativetask/NativeMapOutputCollectorDelegator.java       | 7 ++++---
 .../org/apache/hadoop/mapred/nativetask/NativeRuntime.java  | 7 ++++---
 .../java/org/apache/hadoop/mapred/nativetask/Platforms.java | 6 +++---
 .../hadoop/mapred/nativetask/StatusReportChecker.java       | 7 ++++---
 .../hadoop/mapred/nativetask/handlers/BufferPuller.java     | 6 +++---
 .../hadoop/mapred/nativetask/handlers/BufferPushee.java     | 6 +++---
 .../hadoop/mapred/nativetask/handlers/BufferPusher.java     | 6 +++---
 .../hadoop/mapred/nativetask/handlers/CombinerHandler.java  | 7 ++++---
 .../nativetask/handlers/NativeCollectorOnlyHandler.java     | 7 ++++---
 .../apache/hadoop/mapred/nativetask/serde/KVSerializer.java | 7 +++----
 .../apache/hadoop/mapred/nativetask/util/OutputUtil.java    | 6 +++---
 .../mapred/nativetask/combinertest/LargeKVCombinerTest.java | 7 ++++---
 .../hadoop/mapred/nativetask/combinertest/WordCount.java    | 6 +++---
 .../org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java   | 6 +++---
 .../org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java  | 7 ++++---
 .../apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java | 6 +++---
 .../hadoop/mapred/nativetask/kvtest/TestInputFile.java      | 7 ++++---
 .../testutil/EnforceNativeOutputCollectorDelegator.java     | 7 ++++---
 20 files changed, 72 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
index da6f252..d66d2d2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/HadoopPlatform.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.mapred.nativetask;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.ByteWritable;
@@ -36,10 +34,13 @@ import org.apache.hadoop.io.VLongWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.nativetask.serde.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 public class HadoopPlatform extends Platform {
-  private static final Log LOG = LogFactory.getLog(HadoopPlatform.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(HadoopPlatform.class);
 
   public HadoopPlatform() throws IOException {
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
index 9458c6b..2ec0643 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapred.nativetask;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
@@ -31,13 +29,16 @@ import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
 import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer;
 import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
 import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * used to create channel, transfer data and command between Java and native
  */
 @InterfaceAudience.Private
 public class NativeBatchProcessor implements INativeHandler {
-  private static Log LOG = LogFactory.getLog(NativeBatchProcessor.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NativeBatchProcessor.class);
 
   private final String nativeHandlerName;
   private long nativeHandlerAddr;
@@ -128,7 +129,7 @@ public class NativeBatchProcessor implements INativeHandler {
       NativeRuntime.releaseNativeObject(nativeHandlerAddr);
       nativeHandlerAddr = 0;
     }
-    IOUtils.cleanup(LOG, in);
+    IOUtils.cleanupWithLogger(LOG, in);
     in = null;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
index 5a516c4..b4755fd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeMapOutputCollectorDelegator.java
@@ -21,8 +21,6 @@ import java.io.IOException;
 
 import com.google.common.base.Charsets;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.RawComparator;
 import org.apache.hadoop.mapred.InvalidJobConfException;
@@ -36,6 +34,8 @@ import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.util.QuickSort;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * native map output collector wrapped in Java interface
@@ -43,7 +43,8 @@ import org.apache.hadoop.util.QuickSort;
 @InterfaceAudience.Private
 public class NativeMapOutputCollectorDelegator<K, V> implements MapOutputCollector<K, V> {
 
-  private static Log LOG = LogFactory.getLog(NativeMapOutputCollectorDelegator.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NativeMapOutputCollectorDelegator.class);
   private JobConf job;
   private NativeCollectorOnlyHandler<K, V> handler;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
index cc9adba..8d71ad4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeRuntime.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapred.nativetask;
 import java.io.IOException;
 
 import com.google.common.base.Charsets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataInputBuffer;
@@ -33,6 +31,8 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.Task.TaskReporter;
 import org.apache.hadoop.mapred.nativetask.util.ConfigUtil;
 import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class stands for the native runtime It has three functions:
@@ -42,7 +42,8 @@ import org.apache.hadoop.util.VersionInfo;
  */
 @InterfaceAudience.Private
 public class NativeRuntime {
-  private static Log LOG = LogFactory.getLog(NativeRuntime.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NativeRuntime.class);
   private static boolean nativeLibraryLoaded = false;
 
   private static Configuration conf = new Configuration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
index 9fad3a5..350449d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/Platforms.java
@@ -20,13 +20,13 @@ package org.apache.hadoop.mapred.nativetask;
 import java.io.IOException;
 import java.util.ServiceLoader;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.nativetask.serde.INativeSerializer;
 import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -37,7 +37,7 @@ import org.apache.hadoop.mapred.nativetask.serde.NativeSerialization;
 @InterfaceAudience.Private
 public class Platforms {
 
-  private static final Log LOG = LogFactory.getLog(Platforms.class);
+  private static final Logger LOG = LoggerFactory.getLogger(Platforms.class);
   private static final ServiceLoader<Platform> platforms = ServiceLoader.load(Platform.class);
   
   public static void init(Configuration conf) throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
index 1e76d39..6a437b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/StatusReportChecker.java
@@ -19,11 +19,11 @@ package org.apache.hadoop.mapred.nativetask;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.Task.TaskReporter;
 import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Will periodically check status from native and report to MR framework.
@@ -31,7 +31,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
  */
 class StatusReportChecker implements Runnable {
 
-  private static Log LOG = LogFactory.getLog(StatusReportChecker.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(StatusReportChecker.class);
   public static final int INTERVAL = 1000; // milliseconds
 
   private Thread checker;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
index b5dff92..6c43a65 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPuller.java
@@ -21,8 +21,6 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.mapred.RawKeyValueIterator;
@@ -33,6 +31,8 @@ import org.apache.hadoop.mapred.nativetask.buffer.BufferType;
 import org.apache.hadoop.mapred.nativetask.buffer.ByteBufferDataReader;
 import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
 import org.apache.hadoop.util.Progress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * actively signal a {@link BufferPullee} to load data into buffer and receive
@@ -40,7 +40,7 @@ import org.apache.hadoop.util.Progress;
 @InterfaceAudience.Private
 public class BufferPuller implements RawKeyValueIterator, DataReceiver {
   
-  private static Log LOG = LogFactory.getLog(BufferPuller.class);
+  private static final Logger LOG = LoggerFactory.getLogger(BufferPuller.class);
 
   public final static int KV_HEADER_LENGTH = Constants.SIZEOF_KV_LENGTH;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
index 090882d..a78789e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPushee.java
@@ -21,8 +21,6 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.mapred.RecordWriter;
 import org.apache.hadoop.mapred.nativetask.Constants;
@@ -31,6 +29,8 @@ import org.apache.hadoop.mapred.nativetask.buffer.ByteBufferDataReader;
 import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer;
 import org.apache.hadoop.mapred.nativetask.serde.KVSerializer;
 import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * collect data when signaled
@@ -38,7 +38,7 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
 @InterfaceAudience.Private
 public class BufferPushee<OK, OV> implements Closeable {
 
-  private static Log LOG = LogFactory.getLog(BufferPushee.class);
+  private static final Logger LOG = LoggerFactory.getLogger(BufferPushee.class);
   
   public final static int KV_HEADER_LENGTH = Constants.SIZEOF_KV_LENGTH;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
index 454127c..787e63f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/BufferPusher.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred.nativetask.handlers;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.nativetask.NativeDataTarget;
@@ -29,6 +27,8 @@ import org.apache.hadoop.mapred.nativetask.buffer.ByteBufferDataWriter;
 import org.apache.hadoop.mapred.nativetask.serde.IKVSerializer;
 import org.apache.hadoop.mapred.nativetask.serde.KVSerializer;
 import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * actively push data into a buffer and signal a {@link BufferPushee} to collect it
@@ -36,7 +36,7 @@ import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
 @InterfaceAudience.Private
 public class BufferPusher<K, V> implements OutputCollector<K, V> {
   
-  private static Log LOG = LogFactory.getLog(BufferPusher.class);
+  private static final Logger LOG = LoggerFactory.getLogger(BufferPusher.class);
 
   private final SizedWritable<K> tmpInputKey;
   private final SizedWritable<V> tmpInputValue;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
index fcaeca1..78ae00d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/CombinerHandler.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.mapred.nativetask.handlers;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.Counters.Counter;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Task.CombinerRunner;
@@ -36,10 +34,13 @@ import org.apache.hadoop.mapred.nativetask.serde.SerializationFramework;
 import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 class CombinerHandler<K, V> implements ICombineHandler, CommandDispatcher {
   public static final String NAME = "NativeTask.CombineHandler";
-  private static Log LOG = LogFactory.getLog(NativeCollectorOnlyHandler.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NativeCollectorOnlyHandler.class);
   public static final Command LOAD = new Command(1, "Load");
   public static final Command COMBINE = new Command(4, "Combine");
   public final CombinerRunner<K, V> combinerRunner;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
index 34238ae..8bf4d8b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/handlers/NativeCollectorOnlyHandler.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapred.nativetask.handlers;
 import java.io.Closeable;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -37,6 +35,8 @@ import org.apache.hadoop.mapred.nativetask.TaskContext;
 import org.apache.hadoop.mapred.nativetask.util.NativeTaskOutput;
 import org.apache.hadoop.mapred.nativetask.util.OutputUtil;
 import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Java Record Reader + Java Mapper + Native Collector
@@ -46,7 +46,8 @@ import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer;
 public class NativeCollectorOnlyHandler<K, V> implements CommandDispatcher, Closeable {
 
   public static final String NAME = "NativeTask.MCollectorOutputHandler";
-  private static Log LOG = LogFactory.getLog(NativeCollectorOnlyHandler.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NativeCollectorOnlyHandler.class);
   public static final Command GET_OUTPUT_PATH =
       new Command(100, "GET_OUTPUT_PATH");
   public static final Command GET_OUTPUT_INDEX_PATH =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
index e6609b8..4485ed0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/serde/KVSerializer.java
@@ -20,21 +20,20 @@ package org.apache.hadoop.mapred.nativetask.serde;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.nativetask.Constants;
 import org.apache.hadoop.mapred.nativetask.buffer.DataInputStream;
 import org.apache.hadoop.mapred.nativetask.buffer.DataOutputStream;
 import org.apache.hadoop.mapred.nativetask.util.SizedWritable;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 @InterfaceAudience.Private
 public class KVSerializer<K, V> implements IKVSerializer {
 
-  private static final Log LOG = LogFactory.getLog(KVSerializer.class);
+  private static final Logger LOG = LoggerFactory.getLogger(KVSerializer.class);
   
   public static final int KV_HEAD_LENGTH = Constants.SIZEOF_KV_LENGTH;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
index dd45ede..9809c8e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/util/OutputUtil.java
@@ -20,15 +20,15 @@ package org.apache.hadoop.mapred.nativetask.util;
 
 import java.lang.reflect.Constructor;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @InterfaceAudience.Private
 public class OutputUtil {
 
-  private static Log LOG = LogFactory.getLog(OutputUtil.class);
+  private static final Logger LOG = LoggerFactory.getLogger(OutputUtil.class);
   public static final String NATIVE_TASK_OUTPUT_MANAGER = "nativetask.output.manager";
 
   public static NativeTaskOutput createNativeTaskOutput(Configuration conf, String id) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
index a4eecbe..299acbd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/LargeKVCombinerTest.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred.nativetask.combinertest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -41,11 +39,14 @@ import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
 public class LargeKVCombinerTest {
-  private static final Log LOG = LogFactory.getLog(LargeKVCombinerTest.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(LargeKVCombinerTest.class);
 
   @Before
   public void startUp() throws Exception {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
index c022848..c38f7e8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/combinertest/WordCount.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.mapred.nativetask.combinertest;
 import java.io.IOException;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
@@ -33,10 +31,12 @@ import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.GenericOptionsParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class WordCount {
 
-  private static Log LOG = LogFactory.getLog(WordCount.class);
+  private static final Logger LOG = LoggerFactory.getLogger(WordCount.class);
   
   public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
index 3b4c9c0..802259f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVJob.java
@@ -23,8 +23,6 @@ import java.util.zip.CRC32;
 
 import com.google.common.primitives.Longs;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -37,11 +35,13 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.StopWatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class KVJob {
   public static final String INPUTPATH = "nativetask.kvtest.inputfile.path";
   public static final String OUTPUTPATH = "nativetask.kvtest.outputfile.path";
-  private static final Log LOG = LogFactory.getLog(KVJob.class);
+  private static final Logger LOG = LoggerFactory.getLogger(KVJob.class);
   Job job = null;
 
   public static class ValueMapper<KTYPE, VTYPE> extends Mapper<KTYPE, VTYPE, KTYPE, VTYPE> {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
index 6b658ac..5364401 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/KVTest.java
@@ -23,8 +23,6 @@ import static org.junit.Assert.assertTrue;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -41,13 +39,16 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
 
 @RunWith(Parameterized.class)
 public class KVTest {
-  private static final Log LOG = LogFactory.getLog(KVTest.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(KVTest.class);
 
   private static Configuration nativekvtestconf = ScenarioConfiguration.getNativeConfiguration();
   private static Configuration hadoopkvtestconf = ScenarioConfiguration.getNormalConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
index 5f1619e..4a7b963 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/LargeKVTest.java
@@ -22,8 +22,6 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -38,9 +36,11 @@ import org.apache.hadoop.util.NativeCodeLoader;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class LargeKVTest {
-  private static final Log LOG = LogFactory.getLog(LargeKVTest.class);
+  private static final Logger LOG = LoggerFactory.getLogger(LargeKVTest.class);
 
   @Before
   public void startUp() throws Exception {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
index 04db67a..4cf534b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/kvtest/TestInputFile.java
@@ -21,8 +21,6 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,10 +38,13 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.nativetask.testutil.BytesFactory;
 import org.apache.hadoop.mapred.nativetask.testutil.ScenarioConfiguration;
 import org.apache.hadoop.mapred.nativetask.testutil.TestConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 public class TestInputFile {
-	private static Log LOG = LogFactory.getLog(TestInputFile.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestInputFile.class);
 
   public static class KVSizeScope {
     private static final int DefaultMinNum = 1;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2e8a5c2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
index 7212236..d38056d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/testutil/EnforceNativeOutputCollectorDelegator.java
@@ -19,13 +19,14 @@ package org.apache.hadoop.mapred.nativetask.testutil;
 
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapred.nativetask.NativeMapOutputCollectorDelegator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class EnforceNativeOutputCollectorDelegator<K, V>
   extends NativeMapOutputCollectorDelegator<K, V> {
-  private static final Log LOG = LogFactory.getLog(EnforceNativeOutputCollectorDelegator.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(EnforceNativeOutputCollectorDelegator.class);
   private boolean nativetaskloaded = false;
 
   @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[28/50] hadoop git commit: YARN-7274. Ability to disable elasticity at leaf queue level. (Zian Chen via wangda)

Posted by xy...@apache.org.
YARN-7274. Ability to disable elasticity at leaf queue level. (Zian Chen via wangda)

Change-Id: Ic8d43e297f0f5de788b562f7eff8106c5c35e8d2


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74665e3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74665e3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74665e3a

Branch: refs/heads/HDFS-7240
Commit: 74665e3a7d7f05644d9a5abad5a3f2d47597d6c8
Parents: ef7d334
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:07:56 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:07:56 2017 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/CSQueueUtils.java        |  16 +-
 .../TestNodeLabelContainerAllocation.java       | 163 ++++++++++++++++++
 .../scheduler/capacity/TestQueueParsing.java    | 168 ++++++++++++++++++-
 3 files changed, 329 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74665e3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 6daca51..51e5b17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -66,19 +66,9 @@ class CSQueueUtils {
   private static void capacitiesSanityCheck(String queueName,
       QueueCapacities queueCapacities) {
     for (String label : queueCapacities.getExistingNodeLabels()) {
-      float capacity = queueCapacities.getCapacity(label);
-      float maximumCapacity = queueCapacities.getMaximumCapacity(label);
-      if (capacity > maximumCapacity) {
-        throw new IllegalArgumentException("Illegal queue capacity setting, "
-            + "(capacity=" + capacity + ") > (maximum-capacity="
-            + maximumCapacity + "). When label=[" + label + "]");
-      }
-     
-      // Actually, this may not needed since we have verified capacity <=
-      // maximumCapacity. And the way we compute absolute capacity (abs(x) =
-      // cap(x) * cap(x.parent) * ...) is a monotone increasing function. But
-      // just keep it here to make sure our compute abs capacity method works
-      // correctly. 
+      // The only thing we should care about is absolute capacity <=
+      // absolute max capacity otherwise the absolute max capacity is
+      // no longer an absolute maximum.
       float absCapacity = queueCapacities.getAbsoluteCapacity(label);
       float absMaxCapacity = queueCapacities.getAbsoluteMaximumCapacity(label);
       if (absCapacity > absMaxCapacity) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74665e3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 71fddfc..1836919 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -2083,4 +2083,167 @@ public class TestNodeLabelContainerAllocation {
 
     rm1.close();
   }
+
+  @Test
+  public void testQueueMetricsWithLabelsDisableElasticity() throws Exception {
+    /**
+     * Test case: have a following queue structure:
+     *
+     * <pre>
+     *
+     *          root
+     *        /      \
+     *       a        b
+     *      (x)      (x)
+     *      / \
+     *     a1 a2
+     *    (x) (x)
+     * </pre>
+     *
+     * a/b can access x, both of them has max-capacity-on-x = 50
+     *
+     * When doing non-exclusive allocation, app in a (or b) can use 100% of x
+     * resource.
+     */
+
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        this.conf);
+
+    // Define top-level queues
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] { "a", "b" });
+    csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+
+    final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
+    csConf.setCapacity(queueA, 50);
+    csConf.setMaximumCapacity(queueA, 100);
+    csConf.setAccessibleNodeLabels(queueA, toSet("x"));
+    csConf.setCapacityByLabel(queueA, "x", 50);
+    csConf.setMaximumCapacityByLabel(queueA, "x", 100);
+    final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
+    csConf.setCapacity(queueB, 50);
+    csConf.setMaximumCapacity(queueB, 100);
+    csConf.setAccessibleNodeLabels(queueB, toSet("x"));
+    csConf.setCapacityByLabel(queueB, "x", 50);
+    csConf.setMaximumCapacityByLabel(queueB, "x", 100);
+
+    // Define 2nd-level queues
+    csConf.setQueues(queueA, new String[] { "a1",
+        "a2"});
+
+    final String A1 = queueA + ".a1";
+    csConf.setCapacity(A1, 20);
+    csConf.setMaximumCapacity(A1, 60);
+    csConf.setAccessibleNodeLabels(A1, toSet("x"));
+    csConf.setCapacityByLabel(A1, "x", 60);
+    csConf.setMaximumCapacityByLabel(A1, "x", 30);
+
+    final String A2 = queueA + ".a2";
+    csConf.setCapacity(A2, 80);
+    csConf.setMaximumCapacity(A2, 40);
+    csConf.setAccessibleNodeLabels(A2, toSet("x"));
+    csConf.setCapacityByLabel(A2, "x", 40);
+    csConf.setMaximumCapacityByLabel(A2, "x", 20);
+
+    // set node -> label
+    mgr.addToCluserNodeLabels(
+        ImmutableSet.of(NodeLabel.newInstance("x", false)));
+    mgr.addLabelsToNode(
+        ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+
+    // inject node label manager
+    MockRM rm1 = new MockRM(csConf) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+
+    rm1.getRMContext().setNodeLabelManager(mgr);
+    rm1.start();
+    MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // label = x
+    // app1 -> a1
+    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+    // app1 asks for 6 partition=x containers
+    am1.allocate("*", 1 * GB, 6, new ArrayList<ContainerId>(), "x");
+
+    // NM1 do 50 heartbeats
+    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
+        cs.getApplicationAttempt(am1.getApplicationAttemptId()));
+
+    SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
+        .getNodeReport(nm1.getNodeId());
+    Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(14 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Try to launch app2 in a2, asked 2GB, should success
+    // app2 -> a2
+    RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x");
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+    // app2 asks for 4 partition=x containers
+    am2.allocate("*", 1 * GB, 4, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
+        cs.getApplicationAttempt(am2.getApplicationAttemptId()));
+
+    reportNm1 = rm1.getResourceScheduler()
+        .getNodeReport(nm1.getNodeId());
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Kill all apps in queue a2
+    cs.killAllAppsInQueue("a2");
+    rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
+    rm1.waitForAppRemovedFromScheduler(app2.getApplicationId());
+
+    // Try to launch app3 in a2, asked 6GB, should fail
+    // app3 -> a2
+    RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x");
+    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
+
+    am3.allocate("*", 1 * GB, 6, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    // app3 cannot preempt more resources restricted by disable elasticity
+    checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
+        cs.getApplicationAttempt(am3.getApplicationAttemptId()));
+
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Kill all apps in queue a1
+    cs.killAllAppsInQueue("a1");
+    rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+    rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
+
+    // app4 -> a1, try to allocate more than 6GB resource, should fail
+    RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x");
+    MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm1);
+
+    // app3 asks for 7 partition=x containers
+    am4.allocate("*", 1 * GB, 7, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+
+    // app4 should only gets 6GB resource in partition=x
+    // since elasticity is disabled
+    checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
+        cs.getApplicationAttempt(am4.getApplicationAttemptId()));
+
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    rm1.close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74665e3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 5d167c7..add14ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -365,7 +365,76 @@ public class TestQueueParsing {
     conf.setCapacityByLabel(B3, "red", 25);
     conf.setCapacityByLabel(B3, "blue", 25);
   }
-  
+
+  private void setupQueueConfigurationWithLabelsAndReleaseCheck
+      (CapacitySchedulerConfiguration conf) {
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100);
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100);
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    // The cap <= max-cap check is not needed
+    conf.setCapacity(A, 50);
+    conf.setMaximumCapacity(A, 100);
+
+    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+    conf.setCapacity(B, 50);
+    conf.setMaximumCapacity(B, 100);
+
+    LOG.info("Setup top-level queues");
+
+    // Define 2nd-level queues
+    final String A1 = A + ".a1";
+    final String A2 = A + ".a2";
+    conf.setQueues(A, new String[] {"a1", "a2"});
+    conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue"));
+    conf.setCapacityByLabel(A, "red", 50);
+    conf.setMaximumCapacityByLabel(A, "red", 100);
+    conf.setCapacityByLabel(A, "blue", 30);
+    conf.setMaximumCapacityByLabel(A, "blue", 50);
+
+    conf.setCapacity(A1, 60);
+    conf.setMaximumCapacity(A1, 60);
+    conf.setCapacityByLabel(A1, "red", 60);
+    conf.setMaximumCapacityByLabel(A1, "red", 30);
+    conf.setCapacityByLabel(A1, "blue", 100);
+    conf.setMaximumCapacityByLabel(A1, "blue", 100);
+
+    conf.setCapacity(A2, 40);
+    conf.setMaximumCapacity(A2, 85);
+    conf.setAccessibleNodeLabels(A2, ImmutableSet.of("red"));
+    conf.setCapacityByLabel(A2, "red", 40);
+    conf.setMaximumCapacityByLabel(A2, "red", 60);
+
+    final String B1 = B + ".b1";
+    final String B2 = B + ".b2";
+    final String B3 = B + ".b3";
+    conf.setQueues(B, new String[] {"b1", "b2", "b3"});
+    conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue"));
+    conf.setCapacityByLabel(B, "red", 50);
+    conf.setMaximumCapacityByLabel(B, "red", 100);
+    conf.setCapacityByLabel(B, "blue", 70);
+    conf.setMaximumCapacityByLabel(B, "blue", 100);
+
+    conf.setCapacity(B1, 10);
+    conf.setMaximumCapacity(B1, 10);
+    conf.setCapacityByLabel(B1, "red", 60);
+    conf.setMaximumCapacityByLabel(B1, "red", 30);
+    conf.setCapacityByLabel(B1, "blue", 50);
+    conf.setMaximumCapacityByLabel(B1, "blue", 100);
+
+    conf.setCapacity(B2, 80);
+    conf.setMaximumCapacity(B2, 40);
+    conf.setCapacityByLabel(B2, "red", 30);
+    conf.setCapacityByLabel(B2, "blue", 25);
+
+    conf.setCapacity(B3, 10);
+    conf.setMaximumCapacity(B3, 25);
+    conf.setCapacityByLabel(B3, "red", 10);
+    conf.setCapacityByLabel(B3, "blue", 25);
+  }
+
   private void setupQueueConfigurationWithLabelsInherit(
       CapacitySchedulerConfiguration conf) {
     // Define top-level queues
@@ -472,7 +541,7 @@ public class TestQueueParsing {
     // queue-B2 inherits "red"/"blue"
     Assert.assertTrue(capacityScheduler.getQueue("b2")
         .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
-    
+
     // check capacity of A2
     CSQueue qA2 = capacityScheduler.getQueue("a2");
     Assert.assertEquals(0.7, qA2.getCapacity(), DELTA);
@@ -481,7 +550,7 @@ public class TestQueueParsing {
     Assert.assertEquals(0.25, qA2.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
     Assert.assertEquals(0.1275, qA2.getAbsoluteMaximumCapacity(), DELTA);
     Assert.assertEquals(0.3, qA2.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
-    
+
     // check capacity of B3
     CSQueue qB3 = capacityScheduler.getQueue("b3");
     Assert.assertEquals(0.18, qB3.getAbsoluteCapacity(), DELTA);
@@ -489,7 +558,71 @@ public class TestQueueParsing {
     Assert.assertEquals(0.35, qB3.getAbsoluteMaximumCapacity(), DELTA);
     Assert.assertEquals(1, qB3.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
   }
-  
+
+  private void checkQueueLabelsWithLeafQueueDisableElasticity
+      (CapacityScheduler capacityScheduler) {
+    // queue-A is red, blue
+    Assert.assertTrue(capacityScheduler.getQueue("a").getAccessibleNodeLabels()
+        .containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-A1 inherits A's configuration
+    Assert.assertTrue(capacityScheduler.getQueue("a1")
+        .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-A2 is "red"
+    Assert.assertEquals(1, capacityScheduler.getQueue("a2")
+        .getAccessibleNodeLabels().size());
+    Assert.assertTrue(capacityScheduler.getQueue("a2")
+        .getAccessibleNodeLabels().contains("red"));
+
+    // queue-B is "red"/"blue"
+    Assert.assertTrue(capacityScheduler.getQueue("b").getAccessibleNodeLabels()
+        .containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-B2 inherits "red"/"blue"
+    Assert.assertTrue(capacityScheduler.getQueue("b2")
+        .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
+
+    // check capacity of A2
+    CSQueue qA2 = capacityScheduler.getQueue("a2");
+    Assert.assertEquals(0.4, qA2.getCapacity(), DELTA);
+    Assert.assertEquals(0.4, qA2.getQueueCapacities()
+        .getCapacity("red"), DELTA);
+    Assert.assertEquals(0.2, qA2.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.2, qA2.getQueueCapacities()
+        .getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.85, qA2.getAbsoluteMaximumCapacity(), DELTA);
+    Assert.assertEquals(0.6, qA2.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("red"), DELTA);
+
+    // check disable elasticity at leaf queue level without label
+    CSQueue qB2 = capacityScheduler.getQueue("b2");
+    Assert.assertEquals(0.4, qB2.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.4, qB2.getAbsoluteMaximumCapacity(), DELTA);
+
+    // check disable elasticity at leaf queue level with label
+    CSQueue qA1 = capacityScheduler.getQueue("a1");
+    Assert.assertEquals(0.3, qA1.getQueueCapacities().
+        getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.3, qA1.getQueueCapacities().
+        getAbsoluteMaximumCapacity("red"), DELTA);
+
+    CSQueue qB1 = capacityScheduler.getQueue("b1");
+    Assert.assertEquals(0.3, qB1.getQueueCapacities()
+        .getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.3, qB1.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("red"), DELTA);
+
+    // check capacity of B3
+    CSQueue qB3 = capacityScheduler.getQueue("b3");
+    Assert.assertEquals(0.05, qB3.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.175, qB3.getQueueCapacities()
+        .getAbsoluteCapacity("blue"), DELTA);
+    Assert.assertEquals(0.25, qB3.getAbsoluteMaximumCapacity(), DELTA);
+    Assert.assertEquals(1, qB3.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("blue"), DELTA);
+  }
+
   private void
       checkQueueLabelsInheritConfig(CapacityScheduler capacityScheduler) {
     // queue-A is red, blue
@@ -514,7 +647,7 @@ public class TestQueueParsing {
   @Test
   public void testQueueParsingWithLabels() throws IOException {
     nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue"));
-    
+
     YarnConfiguration conf = new YarnConfiguration();
     CapacitySchedulerConfiguration csConf =
         new CapacitySchedulerConfiguration(conf);
@@ -534,6 +667,31 @@ public class TestQueueParsing {
     checkQueueLabels(capacityScheduler);
     ServiceOperations.stopQuietly(capacityScheduler);
   }
+
+  @Test
+  public void testQueueParsingWithLeafQueueDisableElasticity()
+      throws IOException {
+    nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity
+        (ImmutableSet.of("red", "blue"));
+
+    YarnConfiguration conf = new YarnConfiguration();
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration(conf);
+    setupQueueConfigurationWithLabelsAndReleaseCheck(csConf);
+    CapacityScheduler capacityScheduler = new CapacityScheduler();
+    RMContextImpl rmContext =
+        new RMContextImpl(null, null, null, null, null, null,
+            new RMContainerTokenSecretManager(csConf),
+            new NMTokenSecretManagerInRM(csConf),
+            new ClientToAMTokenSecretManagerInRM(), null);
+    rmContext.setNodeLabelManager(nodeLabelManager);
+    capacityScheduler.setConf(csConf);
+    capacityScheduler.setRMContext(rmContext);
+    capacityScheduler.init(csConf);
+    capacityScheduler.start();
+    checkQueueLabelsWithLeafQueueDisableElasticity(capacityScheduler);
+    ServiceOperations.stopQuietly(capacityScheduler);
+  }
   
   @Test
   public void testQueueParsingWithLabelsInherit() throws IOException {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[43/50] hadoop git commit: HDFS-12891. Do not invalidate blocks if toInvalidate is empty. Contributed by Zsolt Venczel.

Posted by xy...@apache.org.
HDFS-12891. Do not invalidate blocks if toInvalidate is empty. Contributed by Zsolt Venczel.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55fc2d64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55fc2d64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55fc2d64

Branch: refs/heads/HDFS-7240
Commit: 55fc2d6485702a99c6d4bb261a720d1f0498af2b
Parents: 5cd1056
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Mon Dec 11 16:42:25 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Mon Dec 11 16:43:03 2017 -0800

----------------------------------------------------------------------
 .../hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fc2d64/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
index 75561ca..ff68c02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
@@ -293,10 +293,12 @@ class InvalidateBlocks {
       getBlocksToInvalidateByLimit(nodeToECBlocks.get(dn),
           toInvalidate, numECBlocks, remainingLimit);
     }
-    if (toInvalidate.size() > 0 && getBlockSetsSize(dn) == 0) {
-      remove(dn);
+    if (toInvalidate.size() > 0) {
+      if (getBlockSetsSize(dn) == 0) {
+        remove(dn);
+      }
+      dn.addBlocksToBeInvalidated(toInvalidate);
     }
-    dn.addBlocksToBeInvalidated(toInvalidate);
     return toInvalidate;
   }
   


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[17/50] hadoop git commit: YARN-7533. Documentation for absolute resource support in Capacity Scheduler. Contributed by Sunil G.

Posted by xy...@apache.org.
YARN-7533. Documentation for absolute resource support in Capacity Scheduler. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/532bbf46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/532bbf46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/532bbf46

Branch: refs/heads/HDFS-7240
Commit: 532bbf4602e418276961cfc4f6b5e3f70e9cedc0
Parents: 4db4a4a
Author: Sunil G <su...@apache.org>
Authored: Wed Nov 29 22:57:54 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../src/site/markdown/CapacityScheduler.md                | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/532bbf46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 0449e88..86d9167 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -64,6 +64,8 @@ The `CapacityScheduler` supports the following features:
 
 * **Priority Scheduling** - This feature allows applications to be submitted and scheduled with different priorities. Higher integer value indicates higher priority for an application. Currently Application priority is supported only for FIFO ordering policy.
 
+* **Absolute Resource Configuration** - Administrators could specify absolute resources to a queue instead of providing percentage based values. This provides better control for admins to configure required amount of resources for a given queue.
+
 Configuration
 -------------
 
@@ -118,14 +120,18 @@ Configuration
 
 | Property | Description |
 |:---- |:---- |
-| `yarn.scheduler.capacity.<queue-path>.capacity` | Queue *capacity* in percentage (%) as a float (e.g. 12.5). The sum of capacities for all queues, at each level, must be equal to 100. Applications in the queue may consume more resources than the queue's capacity if there are free resources, providing elasticity. |
-| `yarn.scheduler.capacity.<queue-path>.maximum-capacity` | Maximum queue capacity in percentage (%) as a float. This limits the *elasticity* for applications in the queue. Defaults to -1 which disables it. |
+| `yarn.scheduler.capacity.<queue-path>.capacity` | Queue *capacity* in percentage (%) as a float (e.g. 12.5) OR as absolute resource queue minimum capacity. The sum of capacities for all queues, at each level, must be equal to 100. However if absolute resource is configured, sum of absolute resources of child queues could be less than it's parent absolute resource capacity. Applications in the queue may consume more resources than the queue's capacity if there are free resources, providing elasticity. |
+| `yarn.scheduler.capacity.<queue-path>.maximum-capacity` | Maximum queue capacity in percentage (%) as a float OR as absolute resource queue maximum capacity. This limits the *elasticity* for applications in the queue. Defaults to -1 which disables it. |
 | `yarn.scheduler.capacity.<queue-path>.minimum-user-limit-percent` | Each queue enforces a limit on the percentage of resources allocated to a user at any given time, if there is demand for resources. The user limit can vary between a minimum and maximum value. The former (the minimum value) is set to this property value and the latter (the maximum value) depends on the number of users who have submitted applications. For e.g., suppose the value of this property is 25. If two users have submitted applications to a queue, no single user can use more than 50% of the queue resources. If a third user submits an application, no single user can use more than 33% of the queue resources. With 4 or more users, no user can use more than 25% of the queues resources. A value of 100 implies no user limits are imposed. The default is 100. Value is specified as a integer. |
 | `yarn.scheduler.capacity.<queue-path>.user-limit-factor` | The multiple of the queue capacity which can be configured to allow a single user to acquire more resources. By default this is set to 1 which ensures that a single user can never take more than the queue's configured capacity irrespective of how idle the cluster is. Value is specified as a float. |
 | `yarn.scheduler.capacity.<queue-path>.maximum-allocation-mb` | The per queue maximum limit of memory to allocate to each container request at the Resource Manager. This setting overrides the cluster configuration `yarn.scheduler.maximum-allocation-mb`. This value must be smaller than or equal to the cluster maximum. |
 | `yarn.scheduler.capacity.<queue-path>.maximum-allocation-vcores` | The per queue maximum limit of virtual cores to allocate to each container request at the Resource Manager. This setting overrides the cluster configuration `yarn.scheduler.maximum-allocation-vcores`. This value must be smaller than or equal to the cluster maximum. |
 | `yarn.scheduler.capacity.<queue-path>.user-settings.<user-name>.weight` | This floating point value is used when calculating the user limit resource values for users in a queue. This value will weight each user more or less than the other users in the queue. For example, if user A should receive 50% more resources in a queue than users B and C, this property will be set to 1.5 for user A.  Users B and C will default to 1.0. |
 
+  * Resource Allocation using Absolute Resources configuration
+
+ `CapacityScheduler` supports configuration of absolute resources instead of providing Queue *capacity* in percentage. As mentioned in above configuration section for `yarn.scheduler.capacity.<queue-path>.capacity` and `yarn.scheduler.capacity.<queue-path>.max-capacity`, administrator could specify an absolute resource value like `[memory=10240,vcores=12]`. This is a valid configuration which indicates 10GB Memory and 12 VCores.
+
   * Running and Pending Application Limits
   
   The `CapacityScheduler` supports the following parameters to control the running and pending applications:


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[04/50] hadoop git commit: MAPREDUCE-6998. Moving logging APIs over to slf4j in hadoop-mapreduce-client-jobclient. Contributed by Gergely Novák.

Posted by xy...@apache.org.
MAPREDUCE-6998. Moving logging APIs over to slf4j in hadoop-mapreduce-client-jobclient. Contributed by Gergely Novák.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4cae977
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4cae977
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4cae977

Branch: refs/heads/HDFS-7240
Commit: d4cae977a2471ad7b8f803617e41b6f94df19c11
Parents: 6cca5b3
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Dec 7 16:21:25 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Dec 7 16:21:25 2017 +0900

----------------------------------------------------------------------
 .../java/org/apache/hadoop/mapred/ClientCache.java   |  6 +++---
 .../apache/hadoop/mapred/ClientServiceDelegate.java  |  7 ++++---
 .../apache/hadoop/mapred/ResourceMgrDelegate.java    |  7 ++++---
 .../java/org/apache/hadoop/mapred/YARNRunner.java    |  6 +++---
 .../java/org/apache/hadoop/fi/ProbabilityModel.java  |  7 ++++---
 .../org/apache/hadoop/fs/AccumulatingReducer.java    |  8 +++++---
 .../test/java/org/apache/hadoop/fs/DFSCIOTest.java   |  6 +++---
 .../org/apache/hadoop/fs/DistributedFSCheck.java     |  7 ++++---
 .../java/org/apache/hadoop/fs/JHLogAnalyzer.java     |  7 ++++---
 .../test/java/org/apache/hadoop/fs/TestDFSIO.java    |  6 +++---
 .../src/test/java/org/apache/hadoop/fs/TestJHLA.java |  7 ++++---
 .../hadoop/fs/loadGenerator/LoadGeneratorMR.java     |  6 +++---
 .../java/org/apache/hadoop/fs/slive/AppendOp.java    |  6 +++---
 .../org/apache/hadoop/fs/slive/ConfigExtractor.java  |  7 ++++---
 .../java/org/apache/hadoop/fs/slive/CreateOp.java    |  6 +++---
 .../java/org/apache/hadoop/fs/slive/DeleteOp.java    |  6 +++---
 .../test/java/org/apache/hadoop/fs/slive/ListOp.java |  6 +++---
 .../java/org/apache/hadoop/fs/slive/MkdirOp.java     |  6 +++---
 .../test/java/org/apache/hadoop/fs/slive/ReadOp.java |  6 +++---
 .../java/org/apache/hadoop/fs/slive/RenameOp.java    |  6 +++---
 .../org/apache/hadoop/fs/slive/ReportWriter.java     |  6 +++---
 .../java/org/apache/hadoop/fs/slive/SleepOp.java     |  6 +++---
 .../java/org/apache/hadoop/fs/slive/SliveMapper.java |  6 +++---
 .../org/apache/hadoop/fs/slive/SliveReducer.java     |  6 +++---
 .../java/org/apache/hadoop/fs/slive/SliveTest.java   |  6 +++---
 .../java/org/apache/hadoop/fs/slive/TestSlive.java   | 10 +++++-----
 .../java/org/apache/hadoop/fs/slive/TruncateOp.java  |  6 +++---
 .../org/apache/hadoop/fs/slive/WeightSelector.java   |  7 ++++---
 .../test/java/org/apache/hadoop/hdfs/NNBench.java    |  7 +++----
 .../org/apache/hadoop/hdfs/NNBenchWithoutMR.java     |  8 ++++----
 .../java/org/apache/hadoop/mapred/BigMapOutput.java  |  8 ++++----
 .../test/java/org/apache/hadoop/mapred/MRBench.java  |  6 +++---
 .../java/org/apache/hadoop/mapred/MiniMRCluster.java | 15 ++++++++-------
 .../hadoop/mapred/MiniMRYarnClusterAdapter.java      |  7 ++++---
 .../org/apache/hadoop/mapred/ReliabilityTest.java    | 13 +++++++------
 .../org/apache/hadoop/mapred/TestBadRecords.java     |  8 ++++----
 .../org/apache/hadoop/mapred/TestClientRedirect.java |  7 ++++---
 .../hadoop/mapred/TestCombineFileInputFormat.java    | 11 +++++------
 .../mapred/TestCombineSequenceFileInputFormat.java   |  8 ++++----
 .../hadoop/mapred/TestCombineTextInputFormat.java    |  8 ++++----
 .../mapred/TestConcatenatedCompressedInput.java      |  9 +++++----
 .../hadoop/mapred/TestFixedLengthInputFormat.java    | 11 ++++++-----
 .../org/apache/hadoop/mapred/TestJobCleanup.java     |  9 +++++----
 .../apache/hadoop/mapred/TestJobSysDirWithDFS.java   |  8 ++++----
 .../hadoop/mapred/TestKeyValueTextInputFormat.java   |  8 +++++---
 .../hadoop/mapred/TestMRTimelineEventHandling.java   |  8 ++++----
 .../org/apache/hadoop/mapred/TestMapProgress.java    |  8 +++++---
 .../apache/hadoop/mapred/TestMiniMRChildTask.java    |  8 ++++----
 .../hadoop/mapred/TestMultiFileInputFormat.java      |  8 +++++---
 .../mapred/TestSequenceFileAsBinaryOutputFormat.java | 13 ++++++-------
 .../org/apache/hadoop/mapred/TestSortedRanges.java   | 12 ++++++------
 .../mapred/TestSpecialCharactersInOutputPath.java    | 15 +++++++--------
 .../org/apache/hadoop/mapred/TestTaskStatus.java     |  4 +---
 .../apache/hadoop/mapred/TestTextInputFormat.java    | 11 +++++------
 .../org/apache/hadoop/mapred/TestYARNRunner.java     | 14 ++++++++------
 .../apache/hadoop/mapred/ThreadedMapBenchmark.java   |  7 ++++---
 .../java/org/apache/hadoop/mapred/UtilsForTests.java |  6 +++---
 .../mapred/jobcontrol/TestLocalJobControl.java       |  9 +++++----
 .../org/apache/hadoop/mapred/pipes/TestPipes.java    |  8 ++++----
 .../org/apache/hadoop/mapreduce/GrowingSleepJob.java | 12 ++++++------
 .../hadoop/mapreduce/JobHistoryFileParser.java       |  7 ++++---
 .../hadoop/mapreduce/JobHistoryFileReplayHelper.java |  9 ++++-----
 .../mapreduce/JobHistoryFileReplayMapperV1.java      |  9 ++++-----
 .../mapreduce/JobHistoryFileReplayMapperV2.java      |  8 ++++----
 .../apache/hadoop/mapreduce/MapReduceTestUtil.java   | 13 ++++---------
 .../hadoop/mapreduce/MiniHadoopClusterManager.java   |  8 ++++----
 .../hadoop/mapreduce/SimpleEntityWriterV1.java       |  7 ++++---
 .../hadoop/mapreduce/SimpleEntityWriterV2.java       |  9 +++++----
 .../org/apache/hadoop/mapreduce/TestCounters.java    | 11 +++++------
 .../org/apache/hadoop/mapreduce/TestLocalRunner.java | 11 ++++++-----
 .../org/apache/hadoop/mapreduce/TestMRJobClient.java |  7 ++++---
 .../apache/hadoop/mapreduce/TestMapCollection.java   | 13 ++++++-------
 .../apache/hadoop/mapreduce/TestValueIterReset.java  |  8 ++++----
 .../hadoop/mapreduce/TimelineEntityConverterV1.java  |  8 ++++----
 .../hadoop/mapreduce/TimelineEntityConverterV2.java  |  8 ++++----
 .../lib/db/TestDataDrivenDBInputFormat.java          |  6 +++---
 .../input/TestCombineSequenceFileInputFormat.java    |  8 ++++----
 .../lib/input/TestCombineTextInputFormat.java        |  8 ++++----
 .../lib/input/TestFixedLengthInputFormat.java        | 12 ++++++------
 .../lib/input/TestMRKeyValueTextInputFormat.java     | 12 +++++-------
 .../lib/input/TestMRSequenceFileInputFilter.java     |  8 ++++----
 .../lib/jobcontrol/TestMapReduceJobControl.java      | 12 ++++++------
 .../TestMRSequenceFileAsBinaryOutputFormat.java      |  8 ++++----
 .../mapreduce/lib/partition/TestKeyFieldHelper.java  | 10 ++++++----
 .../hadoop/mapreduce/security/TestJHSSecurity.java   | 10 +++++-----
 .../hadoop/mapreduce/util/MRAsyncDiskService.java    |  7 ++++---
 .../mapreduce/util/TestMRAsyncDiskService.java       |  9 ++++-----
 .../hadoop/mapreduce/v2/MiniMRYarnCluster.java       |  7 ++++---
 .../v2/TestMRAMWithNonNormalizedCapabilities.java    |  7 ++++---
 .../hadoop/mapreduce/v2/TestMRAppWithCombiner.java   |  7 ++++---
 .../org/apache/hadoop/mapreduce/v2/TestMRJobs.java   |  6 +++---
 .../mapreduce/v2/TestMRJobsWithHistoryService.java   |  8 ++++----
 .../hadoop/mapreduce/v2/TestMRJobsWithProfiler.java  |  8 ++++----
 .../apache/hadoop/mapreduce/v2/TestMROldApiJobs.java |  7 ++++---
 .../org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java |  7 ++++---
 .../mapreduce/v2/TestSpeculativeExecution.java       |  7 ++++---
 .../org/apache/hadoop/mapreduce/v2/TestUberAM.java   |  6 +++---
 .../src/test/java/testjar/UserNamePermission.java    | 13 ++++---------
 98 files changed, 411 insertions(+), 394 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
index 93ea5c4..8268d1e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java
@@ -23,8 +23,6 @@ import java.security.PrivilegedAction;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.mapreduce.JobID;
@@ -35,13 +33,15 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ClientCache {
 
   private final Configuration conf;
   private final ResourceMgrDelegate rm;
 
-  private static final Log LOG = LogFactory.getLog(ClientCache.class);
+  private static final Logger LOG = LoggerFactory.getLogger(ClientCache.class);
 
   private Map<JobID, ClientServiceDelegate> cache = 
       new HashMap<JobID, ClientServiceDelegate>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index 72339e5..792e496 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -29,8 +29,6 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ipc.RPC;
@@ -79,11 +77,14 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 
 public class ClientServiceDelegate {
-  private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ClientServiceDelegate.class);
   private static final String UNAVAILABLE = "N/A";
 
   // Caches for per-user NotRunningJobs

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 94f741a..ac4b73b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -25,8 +25,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -78,11 +76,14 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 
 public class ResourceMgrDelegate extends YarnClient {
-  private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ResourceMgrDelegate.class);
       
   private YarnConfiguration conf;
   private ApplicationSubmissionContext application;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 12a3079..127e1dc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -36,8 +36,6 @@ import java.util.Vector;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
@@ -99,6 +97,8 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -108,7 +108,7 @@ import com.google.common.annotations.VisibleForTesting;
 @SuppressWarnings("unchecked")
 public class YARNRunner implements ClientProtocol {
 
-  private static final Log LOG = LogFactory.getLog(YARNRunner.class);
+  private static final Logger LOG = LoggerFactory.getLogger(YARNRunner.class);
 
   private static final String RACK_GROUP = "rack";
   private static final String NODE_IF_RACK_GROUP = "node1";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fi/ProbabilityModel.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fi/ProbabilityModel.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fi/ProbabilityModel.java
index 48e57f5..ce3f1fa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fi/ProbabilityModel.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fi/ProbabilityModel.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.fi;
 
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is responsible for the decision of when a fault 
@@ -42,7 +42,8 @@ import org.apache.hadoop.conf.Configuration;
  */
 public class ProbabilityModel {
   private static Random generator = new Random();
-  private static final Log LOG = LogFactory.getLog(ProbabilityModel.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ProbabilityModel.class);
 
   static final String FPROB_NAME = "fi.";
   private static final String ALL_PROBABILITIES = FPROB_NAME + "*";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/AccumulatingReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/AccumulatingReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/AccumulatingReducer.java
index 3991d94..f6c2a06 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/AccumulatingReducer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/AccumulatingReducer.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.fs;
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Reducer that accumulates values based on their type.
@@ -47,7 +47,9 @@ public class AccumulatingReducer extends MapReduceBase
   static final String VALUE_TYPE_LONG = "l:";
   static final String VALUE_TYPE_FLOAT = "f:";
   static final String VALUE_TYPE_STRING = "s:";
-  private static final Log LOG = LogFactory.getLog(AccumulatingReducer.class);
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(AccumulatingReducer.class);
   
   protected String hostName;
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
index b01954e..d718556 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DFSCIOTest.java
@@ -28,8 +28,6 @@ import java.io.PrintStream;
 import java.util.Date;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
@@ -38,6 +36,8 @@ import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.mapred.*;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
  /**
  * Distributed i/o benchmark.
@@ -69,7 +69,7 @@ import org.junit.Test;
 @Ignore
 public class DFSCIOTest {
   // Constants
-  private static final Log LOG = LogFactory.getLog(DFSCIOTest.class);
+  private static final Logger LOG = LoggerFactory.getLogger(DFSCIOTest.class);
   private static final int TEST_TYPE_READ = 0;
   private static final int TEST_TYPE_WRITE = 1;
   private static final int TEST_TYPE_CLEANUP = 2;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
index 67ef5d9..8ed2c19 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
@@ -33,8 +33,6 @@ import java.util.Vector;
 
 import junit.framework.TestCase;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
@@ -42,6 +40,8 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.SequenceFile.CompressionType;
 import org.apache.hadoop.mapred.*;
 import org.junit.Ignore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Distributed checkup of the file system consistency.
@@ -56,7 +56,8 @@ import org.junit.Ignore;
 @Ignore
 public class DistributedFSCheck extends TestCase {
   // Constants
-  private static final Log LOG = LogFactory.getLog(DistributedFSCheck.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DistributedFSCheck.class);
   private static final int TEST_TYPE_READ = 0;
   private static final int TEST_TYPE_CLEANUP = 2;
   private static final int DEFAULT_BUFFER_SIZE = 1000000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java
index 91c3c26..5e3e745 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java
@@ -34,8 +34,6 @@ import java.util.Map;
 import java.util.StringTokenizer;
 import java.util.HashMap;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
@@ -46,6 +44,8 @@ import org.apache.hadoop.io.compress.GzipCodec;
 import org.apache.hadoop.mapred.*;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Job History Log Analyzer.
@@ -144,7 +144,8 @@ import org.apache.hadoop.util.StringUtils;
  */
 @SuppressWarnings("deprecation")
 public class JHLogAnalyzer {
-  private static final Log LOG = LogFactory.getLog(JHLogAnalyzer.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JHLogAnalyzer.class);
   // Constants
   private static final String JHLA_ROOT_DIR = 
                             System.getProperty("test.build.data", "stats/JHLA");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index 61485be..68befea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -33,8 +33,6 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.Random;
 import java.util.StringTokenizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -62,6 +60,8 @@ import org.apache.hadoop.util.ToolRunner;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Distributed i/o benchmark.
@@ -92,7 +92,7 @@ import org.junit.Test;
  */
 public class TestDFSIO implements Tool {
   // Constants
-  private static final Log LOG = LogFactory.getLog(TestDFSIO.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestDFSIO.class);
   private static final int DEFAULT_BUFFER_SIZE = 1000000;
   private static final String BASE_FILE_NAME = "test_io_";
   private static final String DEFAULT_RES_FILE_NAME = "TestDFSIO_results.log";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java
index 31950fd..9334a8a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestJHLA.java
@@ -23,11 +23,11 @@ import java.io.FileOutputStream;
 import java.io.OutputStreamWriter;
 import java.io.File;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Test Job History Log Analyzer.
@@ -35,7 +35,8 @@ import org.junit.Test;
  * @see JHLogAnalyzer
  */
 public class TestJHLA {
-  private static final Log LOG = LogFactory.getLog(JHLogAnalyzer.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(JHLogAnalyzer.class);
   private String historyLog = System.getProperty("test.build.data", 
                                   "build/test/data") + "/history/test.log";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
index c47d971..044c77c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGeneratorMR.java
@@ -26,8 +26,6 @@ import java.net.UnknownHostException;
 import java.util.EnumSet;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.CreateFlag;
@@ -50,6 +48,8 @@ import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** The load generator is a tool for testing NameNode behavior under
  * different client loads.
@@ -63,7 +63,7 @@ import org.apache.hadoop.util.ToolRunner;
  *
  */
 public class LoadGeneratorMR extends LoadGenerator {
-  public static final Log LOG = LogFactory.getLog(LoadGenerator.class);
+  public static final Logger LOG = LoggerFactory.getLogger(LoadGenerator.class);
   private static int numMapTasks = 1;
   private String mrOutDir;
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
index 82d221f..b118e6e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/AppendOp.java
@@ -24,12 +24,12 @@ import java.io.OutputStream;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and appends a random amount of bytes
@@ -41,7 +41,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class AppendOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(AppendOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(AppendOp.class);
 
   AppendOp(ConfigExtractor cfg, Random rnd) {
     super(AppendOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ConfigExtractor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ConfigExtractor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ConfigExtractor.java
index ef4e436..2668770 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ConfigExtractor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ConfigExtractor.java
@@ -22,12 +22,12 @@ import java.text.NumberFormat;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.Constants.OperationType;
 import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Simple access layer onto of a configuration object that extracts the slive
@@ -35,7 +35,8 @@ import org.apache.hadoop.util.StringUtils;
  */
 class ConfigExtractor {
 
-  private static final Log LOG = LogFactory.getLog(ConfigExtractor.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ConfigExtractor.class);
 
   private Configuration config;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/CreateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/CreateOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/CreateOp.java
index 7da32c7..d7d2989 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/CreateOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/CreateOp.java
@@ -22,13 +22,13 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and a random number of bytes to create
@@ -42,7 +42,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class CreateOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(CreateOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(CreateOp.class);
 
   private static int DEF_IO_BUFFER_SIZE = 4096;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/DeleteOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/DeleteOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/DeleteOp.java
index 31afd16..0ed425a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/DeleteOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/DeleteOp.java
@@ -23,11 +23,11 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and attempts to delete that file (if it
@@ -39,7 +39,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class DeleteOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(DeleteOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(DeleteOp.class);
 
   DeleteOp(ConfigExtractor cfg, Random rnd) {
     super(DeleteOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ListOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ListOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ListOp.java
index f83a6bd..46377d5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ListOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ListOp.java
@@ -23,12 +23,12 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random directory and attempts to list that
@@ -41,7 +41,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class ListOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(ListOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(ListOp.class);
 
   ListOp(ConfigExtractor cfg, Random rnd) {
     super(ListOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/MkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/MkdirOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/MkdirOp.java
index 585f418..0f24d14 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/MkdirOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/MkdirOp.java
@@ -23,11 +23,11 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random directory and attempts to create that
@@ -40,7 +40,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class MkdirOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(MkdirOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(MkdirOp.class);
 
   MkdirOp(ConfigExtractor cfg, Random rnd) {
     super(MkdirOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReadOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReadOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReadOp.java
index 9683323..d6f29a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReadOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReadOp.java
@@ -24,12 +24,12 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.DataVerifier.VerifyOutput;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and selects a random read size (from
@@ -43,7 +43,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  * number of failures and the amount of time taken to fail
  */
 class ReadOp extends Operation {
-  private static final Log LOG = LogFactory.getLog(ReadOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(ReadOp.class);
 
   ReadOp(ConfigExtractor cfg, Random rnd) {
     super(ReadOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/RenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/RenameOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/RenameOp.java
index 94d6db4..a608a87 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/RenameOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/RenameOp.java
@@ -23,11 +23,11 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and a second random file and attempts
@@ -60,7 +60,7 @@ class RenameOp extends Operation {
     }
   }
 
-  private static final Log LOG = LogFactory.getLog(RenameOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(RenameOp.class);
 
   RenameOp(ConfigExtractor cfg, Random rnd) {
     super(RenameOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReportWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReportWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReportWriter.java
index 1f9abde..873a2ffd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReportWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/ReportWriter.java
@@ -24,8 +24,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Class which provides a report for the given operation output
@@ -48,7 +48,7 @@ class ReportWriter {
   static final String NOT_FOUND = "files_not_found";
   static final String BAD_FILES = "bad_files";
 
-  private static final Log LOG = LogFactory.getLog(ReportWriter.class);
+  private static final Logger LOG = LoggerFactory.getLogger(ReportWriter.class);
 
   private static final String SECTION_DELIM = "-------------";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SleepOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SleepOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SleepOp.java
index 9fc9b30..2b9d54e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SleepOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SleepOp.java
@@ -21,10 +21,10 @@ package org.apache.hadoop.fs.slive;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which sleeps for a given number of milliseconds according to the
@@ -32,7 +32,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class SleepOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(SleepOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SleepOp.class);
 
   SleepOp(ConfigExtractor cfg, Random rnd) {
     super(SleepOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
index e02c5d9..93ee9f9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
@@ -22,8 +22,6 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
 import org.apache.hadoop.io.Text;
@@ -35,6 +33,8 @@ import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The slive class which sets up the mapper to be used which itself will receive
@@ -45,7 +45,7 @@ import org.apache.hadoop.util.StringUtils;
 public class SliveMapper extends MapReduceBase implements
     Mapper<Object, Object, Text, Text> {
 
-  private static final Log LOG = LogFactory.getLog(SliveMapper.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SliveMapper.class);
 
   private static final String OP_TYPE = SliveMapper.class.getSimpleName();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveReducer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveReducer.java
index 323559c..d4f2473 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveReducer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveReducer.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.fs.slive;
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.MapReduceBase;
@@ -30,6 +28,8 @@ import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * The slive reducer which iterates over the given input values and merges them
@@ -38,7 +38,7 @@ import org.apache.hadoop.util.StringUtils;
 public class SliveReducer extends MapReduceBase implements
     Reducer<Text, Text, Text, Text> {
 
-  private static final Log LOG = LogFactory.getLog(SliveReducer.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SliveReducer.class);
 
   private ConfigExtractor config;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveTest.java
index 97360d6..c2918ab 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveTest.java
@@ -30,8 +30,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
@@ -45,6 +43,8 @@ import org.apache.hadoop.mapred.TextOutputFormat;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Slive test entry point + main program
@@ -61,7 +61,7 @@ import org.apache.hadoop.util.ToolRunner;
  */
 public class SliveTest implements Tool {
 
-  private static final Log LOG = LogFactory.getLog(SliveTest.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SliveTest.class);
 
   // ensures the hdfs configurations are loaded if they exist
   static {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TestSlive.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TestSlive.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TestSlive.java
index 25e3340..575cd0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TestSlive.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TestSlive.java
@@ -31,8 +31,6 @@ import java.util.List;
 import java.util.Random;
 import java.util.Set;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -43,13 +41,15 @@ import org.apache.hadoop.fs.slive.DataWriter.GenerateOutput;
 import org.apache.hadoop.util.ToolRunner;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Junit 4 test for slive
  */
 public class TestSlive {
 
-  private static final Log LOG = LogFactory.getLog(TestSlive.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestSlive.class);
 
   private static final Random rnd = new Random(1L);
 
@@ -258,13 +258,13 @@ public class TestSlive {
     DataWriter writer = new DataWriter(rnd);
     FileOutputStream fs = new FileOutputStream(fn);
     GenerateOutput ostat = writer.writeSegment(byteAm, fs);
-    LOG.info(ostat);
+    LOG.info(ostat.toString());
     fs.close();
     assertTrue(ostat.getBytesWritten() == byteAm);
     DataVerifier vf = new DataVerifier();
     FileInputStream fin = new FileInputStream(fn);
     VerifyOutput vfout = vf.verifyFile(byteAm, new DataInputStream(fin));
-    LOG.info(vfout);
+    LOG.info(vfout.toString());
     fin.close();
     assertEquals(vfout.getBytesRead(), byteAm);
     assertTrue(vfout.getChunksDifferent() == 0);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
index 202d807..295b797 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/TruncateOp.java
@@ -22,12 +22,12 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.List;
 import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Operation which selects a random file and truncates a random amount of bytes
@@ -40,7 +40,7 @@ import org.apache.hadoop.fs.slive.OperationOutput.OutputType;
  */
 class TruncateOp extends Operation {
 
-  private static final Log LOG = LogFactory.getLog(TruncateOp.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TruncateOp.class);
 
   TruncateOp(ConfigExtractor cfg, Random rnd) {
     super(TruncateOp.class.getSimpleName(), cfg, rnd);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/WeightSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/WeightSelector.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/WeightSelector.java
index d8acc39..3d80357 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/WeightSelector.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/WeightSelector.java
@@ -26,12 +26,12 @@ import java.util.Map;
 import java.util.Random;
 import java.util.TreeMap;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.slive.Constants.Distribution;
 import org.apache.hadoop.fs.slive.Constants.OperationType;
 import org.apache.hadoop.fs.slive.Weights.UniformWeight;
 import org.apache.hadoop.fs.slive.ObserveableOp.Observer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is the main handler that selects operations to run using the
@@ -47,7 +47,8 @@ class WeightSelector {
     Double weight(int elapsed, int duration);
   }
 
-  private static final Log LOG = LogFactory.getLog(WeightSelector.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(WeightSelector.class);
 
   private static class OperationInfo {
     Integer amountLeft;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
index 29eac43..2346c3c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBench.java
@@ -30,8 +30,6 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -57,6 +55,8 @@ import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This program executes a specified operation that applies load to 
@@ -78,8 +78,7 @@ import org.apache.hadoop.util.ToolRunner;
  */
 
 public class NNBench extends Configured implements Tool {
-  private static final Log LOG = LogFactory.getLog(
-          "org.apache.hadoop.hdfs.NNBench");
+  private static final Logger LOG = LoggerFactory.getLogger(NNBench.class);
   
   private static String CONTROL_DIR_NAME = "control";
   private static String OUTPUT_DIR_NAME = "output";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
index 9b63010..af16177 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/hdfs/NNBenchWithoutMR.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hdfs;
 import java.io.IOException;
 import java.util.Date;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -30,6 +28,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.mapred.JobConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This program executes a specified operation that applies load to 
@@ -45,8 +45,8 @@ import org.apache.hadoop.mapred.JobConf;
  */
 public class NNBenchWithoutMR {
   
-  private static final Log LOG = LogFactory.getLog(
-                                            "org.apache.hadoop.hdfs.NNBench");
+  private static final Logger LOG =
+      LoggerFactory.getLogger(NNBenchWithoutMR.class);
   
   // variable initialzed from command line arguments
   private static long startTime = 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
index 14e32fd..964673b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/BigMapOutput.java
@@ -22,8 +22,6 @@ import java.io.IOException;
 import java.util.Date;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
@@ -37,10 +35,12 @@ import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class BigMapOutput extends Configured implements Tool {
-  public static final Log LOG =
-    LogFactory.getLog(BigMapOutput.class.getName());
+  public static final Logger LOG =
+      LoggerFactory.getLogger(BigMapOutput.class);
   private static Random random = new Random();
   public static String MIN_KEY = "mapreduce.bmo.minkey";
   public static String MIN_VALUE = "mapreduce.bmo.minvalue";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
index 5286e86..5328756 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRBench.java
@@ -24,8 +24,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -33,13 +31,15 @@ import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Runs a job multiple times and takes average of all runs.
  */
 public class MRBench extends Configured implements Tool{
   
-  private static final Log LOG = LogFactory.getLog(MRBench.class);
+  private static final Logger LOG = LoggerFactory.getLogger(MRBench.class);
   private static final String DEFAULT_INPUT_SUB = "mr_input";
   private static final String DEFAULT_OUTPUT_SUB = "mr_output";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
index 2e144414..e7df5b3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRCluster.java
@@ -20,13 +20,13 @@ package org.apache.hadoop.mapred;
 import java.io.IOException;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is an MR2 replacement for older MR1 MiniMRCluster, that was used
@@ -45,7 +45,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class MiniMRCluster {
-  private static final Log LOG = LogFactory.getLog(MiniMRCluster.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MiniMRCluster.class);
 
   private MiniMRClientCluster mrClientCluster;
 
@@ -98,7 +99,7 @@ public class MiniMRCluster {
     try {
       jobConf = new JobConf(mrClientCluster.getConfig());
     } catch (IOException e) {
-      LOG.error(e);
+      LOG.error(e.getMessage());
     }
     return jobConf;
   }
@@ -108,7 +109,7 @@ public class MiniMRCluster {
     try {
       jobConf = new JobConf(mrClientCluster.getConfig());
     } catch (IOException e) {
-      LOG.error(e);
+      LOG.error(e.getMessage());
     }
     return jobConf;
   }
@@ -224,7 +225,7 @@ public class MiniMRCluster {
     try {
       jobConf = new JobConf(mrClientCluster.getConfig());
     } catch (IOException e) {
-      LOG.error(e);
+      LOG.error(e.getMessage());
     }
     return jobConf;
   }
@@ -266,7 +267,7 @@ public class MiniMRCluster {
     try {
       mrClientCluster.stop();
     } catch (IOException e) {
-      LOG.error(e);
+      LOG.error(e.getMessage());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java
index 94d6ff3..4f89840 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java
@@ -18,13 +18,13 @@
 
 package org.apache.hadoop.mapred;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * An adapter for MiniMRYarnCluster providing a MiniMRClientCluster interface.
@@ -34,7 +34,8 @@ public class MiniMRYarnClusterAdapter implements MiniMRClientCluster {
 
   private MiniMRYarnCluster miniMRYarnCluster;
 
-  private static final Log LOG = LogFactory.getLog(MiniMRYarnClusterAdapter.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MiniMRYarnClusterAdapter.class);
 
   public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster) {
     this.miniMRYarnCluster = miniMRYarnCluster;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
index 983a4a7..303857b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
@@ -29,8 +29,6 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
@@ -41,6 +39,8 @@ import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class tests reliability of the framework in the face of failures of
@@ -73,7 +73,8 @@ import org.apache.hadoop.util.ToolRunner;
 public class ReliabilityTest extends Configured implements Tool {
 
   private String dir;
-  private static final Log LOG = LogFactory.getLog(ReliabilityTest.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ReliabilityTest.class);
 
   private void displayUsage() {
     LOG.info("This must be run in only the distributed mode " +
@@ -207,7 +208,7 @@ public class ReliabilityTest extends Configured implements Tool {
               args);
           checkJobExitStatus(status, jobClass);
         } catch (Exception e) {
-          LOG.fatal("JOB " + jobClass + " failed to run");
+          LOG.error("JOB " + jobClass + " failed to run");
           System.exit(-1);
         }
       }
@@ -325,7 +326,7 @@ public class ReliabilityTest extends Configured implements Tool {
           killed = true;
           return;
         } catch (Exception e) {
-          LOG.fatal(StringUtils.stringifyException(e));
+          LOG.error(StringUtils.stringifyException(e));
         }
       }
     }
@@ -495,7 +496,7 @@ public class ReliabilityTest extends Configured implements Tool {
         } catch (InterruptedException ie) {
           killed = true;
         } catch (Exception e) {
-          LOG.fatal(StringUtils.stringifyException(e));
+          LOG.error(StringUtils.stringifyException(e));
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java
index c2d6257..b45a2a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestBadRecords.java
@@ -30,8 +30,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.StringTokenizer;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -41,6 +39,8 @@ import org.apache.hadoop.mapreduce.TaskCounter;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -48,8 +48,8 @@ import static org.junit.Assert.assertNotNull;
 @Ignore
 public class TestBadRecords extends ClusterMapReduceTestCase {
   
-  private static final Log LOG = 
-    LogFactory.getLog(TestBadRecords.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestBadRecords.class);
   
   private static final List<String> MAPPER_BAD_RECORDS = 
     Arrays.asList("hello01","hello04","hello05");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index a9b4626..f97d0a4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -24,8 +24,6 @@ import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.mapreduce.Cluster;
@@ -144,6 +142,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestClientRedirect {
 
@@ -151,7 +151,8 @@ public class TestClientRedirect {
     DefaultMetricsSystem.setMiniClusterMode(true);
   }
 
-  private static final Log LOG = LogFactory.getLog(TestClientRedirect.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestClientRedirect.class);
   private static final String RMADDRESS = "0.0.0.0:8054";
   private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
index de7880d..4ed9eb2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java
@@ -26,16 +26,15 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
 import org.apache.hadoop.mapred.lib.CombineFileSplit;
 import org.apache.hadoop.mapred.lib.CombineFileRecordReader;
-
 import org.junit.Test;
-import static org.junit.Assert.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import static org.junit.Assert.*;
 
 public class TestCombineFileInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestCombineFileInputFormat.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCombineFileInputFormat.class);
   
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null; 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
index 8cdaa80..4f1d6ba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java
@@ -25,8 +25,6 @@ import java.io.IOException;
 import java.util.BitSet;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -36,10 +34,12 @@ import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.mapred.lib.CombineFileSplit;
 import org.apache.hadoop.mapred.lib.CombineSequenceFileInputFormat;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestCombineSequenceFileInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestCombineSequenceFileInputFormat.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCombineSequenceFileInputFormat.class);
 
   private static Configuration conf = new Configuration();
   private static FileSystem localFs = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
index 581e62b..394630c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java
@@ -31,8 +31,6 @@ import java.util.BitSet;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -43,10 +41,12 @@ import org.apache.hadoop.mapred.lib.CombineFileSplit;
 import org.apache.hadoop.mapred.lib.CombineTextInputFormat;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestCombineTextInputFormat {
-  private static final Log LOG =
-    LogFactory.getLog(TestCombineTextInputFormat.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCombineTextInputFormat.class);
 
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
index 15d651d..977d083 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java
@@ -30,8 +30,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.zip.Inflater;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -46,10 +44,13 @@ import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.After;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 @Ignore
 public class TestConcatenatedCompressedInput {
-  private static final Log LOG =
-    LogFactory.getLog(TestConcatenatedCompressedInput.class.getName());
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestConcatenatedCompressedInput.class);
   private static int MAX_LENGTH = 10000;
   private static JobConf defaultConf = new JobConf();
   private static FileSystem localFs = null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
index 8013feb..4864dd0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java
@@ -26,8 +26,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -35,14 +33,18 @@ import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.compress.*;
 import org.apache.hadoop.util.ReflectionUtils;
-
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.junit.Assert.*;
 
 public class TestFixedLengthInputFormat {
 
-  private static Log LOG;
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestFixedLengthInputFormat.class);
+
   private static Configuration defaultConf;
   private static FileSystem localFs; 
   private static Path workDir;
@@ -55,7 +57,6 @@ public class TestFixedLengthInputFormat {
   @BeforeClass
   public static void onlyOnce() {
     try {
-      LOG = LogFactory.getLog(TestFixedLengthInputFormat.class.getName());
       defaultConf = new Configuration();
       defaultConf.set("fs.defaultFS", "file:///");
       localFs = FileSystem.getLocal(defaultConf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
index bf762d9..13f2301 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
@@ -22,9 +22,6 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.Log;
-
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
@@ -36,6 +33,9 @@ import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Logger;
+
 import static org.junit.Assert.*;
 
 /**
@@ -54,7 +54,8 @@ public class TestJobCleanup {
   private static Path emptyInDir = null;
   private static int outDirs = 0;
 
-  private static Log LOG = LogFactory.getLog(TestJobCleanup.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestJobCleanup.class);
 
   @BeforeClass
   public static void setUp() throws IOException {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[15/50] hadoop git commit: YARN-7538. Fix performance regression introduced by Capacity Scheduler absolute min/max resource refactoring. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7538. Fix performance regression introduced by Capacity Scheduler absolute min/max resource refactoring. (Sunil G via wangda)

Change-Id: Ic9bd7e599c56970fe01cb0e1bba6df7d1f77eb29


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7b8cd53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7b8cd53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7b8cd53

Branch: refs/heads/HDFS-7240
Commit: b7b8cd53242da8d47ba4a6d99d906bdb2a1a3494
Parents: 7462c38
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 23 19:52:05 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../scheduler/AbstractResourceUsage.java        | 68 +++++++++++---------
 .../scheduler/capacity/AbstractCSQueue.java     |  4 +-
 .../scheduler/capacity/CSQueueUtils.java        |  8 +--
 .../scheduler/capacity/ParentQueue.java         |  4 +-
 .../capacity/TestCapacityScheduler.java         |  2 +-
 5 files changed, 46 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7b8cd53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
index c295323..3203543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -38,17 +39,20 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 public class AbstractResourceUsage {
   protected ReadLock readLock;
   protected WriteLock writeLock;
-  protected Map<String, UsageByLabel> usages;
+  protected final Map<String, UsageByLabel> usages;
+  private final UsageByLabel noLabelUsages;
   // short for no-label :)
-  private static final String NL = CommonNodeLabelsManager.NO_LABEL;
 
   public AbstractResourceUsage() {
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     readLock = lock.readLock();
     writeLock = lock.writeLock();
 
-    usages = new HashMap<String, UsageByLabel>();
-    usages.put(NL, new UsageByLabel(NL));
+    usages = new HashMap<>();
+
+    // For default label, avoid map for faster access.
+    noLabelUsages = new UsageByLabel();
+    usages.put(CommonNodeLabelsManager.NO_LABEL, noLabelUsages);
   }
 
   // Usage enum here to make implement cleaner
@@ -57,41 +61,40 @@ public class AbstractResourceUsage {
     // be written by ordering policies
     USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
         5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
-            9), EFF_MAX_RESOURCE(
-                10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+            9), EFF_MAX_RESOURCE(10);
 
     private int idx;
 
-    private ResourceType(int value) {
+    ResourceType(int value) {
       this.idx = value;
     }
   }
 
   public static class UsageByLabel {
     // usage by label, contains all UsageType
-    private Resource[] resArr;
+    private final AtomicReferenceArray<Resource> resArr;
 
-    public UsageByLabel(String label) {
-      resArr = new Resource[ResourceType.values().length];
-      for (int i = 0; i < resArr.length; i++) {
-        resArr[i] = Resource.newInstance(0, 0);
-      };
+    public UsageByLabel() {
+      resArr = new AtomicReferenceArray<>(ResourceType.values().length);
+      for (int i = 0; i < resArr.length(); i++) {
+        resArr.set(i, Resource.newInstance(0, 0));
+      }
     }
 
     public Resource getUsed() {
-      return resArr[ResourceType.USED.idx];
+      return resArr.get(ResourceType.USED.idx);
     }
 
     @Override
     public String toString() {
       StringBuilder sb = new StringBuilder();
-      sb.append("{used=" + resArr[0] + "%, ");
-      sb.append("pending=" + resArr[1] + "%, ");
-      sb.append("am_used=" + resArr[2] + "%, ");
-      sb.append("reserved=" + resArr[3] + "%}");
-      sb.append("min_eff=" + resArr[9] + "%, ");
-      sb.append("max_eff=" + resArr[10] + "%}");
-      sb.append("min_effup=" + resArr[11] + "%, ");
+      sb.append("{used=" + resArr.get(ResourceType.USED.idx) + ", ");
+      sb.append("pending=" + resArr.get(ResourceType.PENDING.idx) + ", ");
+      sb.append("am_used=" + resArr.get(ResourceType.AMUSED.idx) + ", ");
+      sb.append("reserved=" + resArr.get(ResourceType.RESERVED.idx) + ", ");
+      sb.append("min_eff=" + resArr.get(ResourceType.EFF_MIN_RESOURCE.idx) + ", ");
+      sb.append(
+          "max_eff=" + resArr.get(ResourceType.EFF_MAX_RESOURCE.idx) + "}");
       return sb.toString();
     }
   }
@@ -104,8 +107,8 @@ public class AbstractResourceUsage {
   }
 
   protected Resource _get(String label, ResourceType type) {
-    if (label == null) {
-      label = RMNodeLabelsManager.NO_LABEL;
+    if (label == null || label.equals(RMNodeLabelsManager.NO_LABEL)) {
+      return normalize(noLabelUsages.resArr.get(type.idx));
     }
 
     try {
@@ -114,7 +117,7 @@ public class AbstractResourceUsage {
       if (null == usage) {
         return Resources.none();
       }
-      return normalize(usage.resArr[type.idx]);
+      return normalize(usage.resArr.get(type.idx));
     } finally {
       readLock.unlock();
     }
@@ -126,7 +129,7 @@ public class AbstractResourceUsage {
       Resource allOfType = Resources.createResource(0);
       for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
         //all usages types are initialized
-        Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+        Resources.addTo(allOfType, usageEntry.getValue().resArr.get(type.idx));
       }
       return allOfType;
     } finally {
@@ -135,11 +138,12 @@ public class AbstractResourceUsage {
   }
 
   private UsageByLabel getAndAddIfMissing(String label) {
-    if (label == null) {
-      label = RMNodeLabelsManager.NO_LABEL;
+    if (label == null || label.equals(RMNodeLabelsManager.NO_LABEL)) {
+      return noLabelUsages;
     }
+
     if (!usages.containsKey(label)) {
-      UsageByLabel u = new UsageByLabel(label);
+      UsageByLabel u = new UsageByLabel();
       usages.put(label, u);
       return u;
     }
@@ -151,7 +155,7 @@ public class AbstractResourceUsage {
     try {
       writeLock.lock();
       UsageByLabel usage = getAndAddIfMissing(label);
-      usage.resArr[type.idx] = res;
+      usage.resArr.set(type.idx, res);
     } finally {
       writeLock.unlock();
     }
@@ -161,7 +165,8 @@ public class AbstractResourceUsage {
     try {
       writeLock.lock();
       UsageByLabel usage = getAndAddIfMissing(label);
-      Resources.addTo(usage.resArr[type.idx], res);
+      usage.resArr.set(type.idx,
+          Resources.add(usage.resArr.get(type.idx), res));
     } finally {
       writeLock.unlock();
     }
@@ -171,7 +176,8 @@ public class AbstractResourceUsage {
     try {
       writeLock.lock();
       UsageByLabel usage = getAndAddIfMissing(label);
-      Resources.subtractFrom(usage.resArr[type.idx], res);
+      usage.resArr.set(type.idx,
+          Resources.subtract(usage.resArr.get(type.idx), res));
     } finally {
       writeLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7b8cd53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 9caf589..140ea5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -932,7 +932,7 @@ public abstract class AbstractCSQueue implements CSQueue {
     queueUsage.incUsed(nodeLabel, resourceToInc);
     CSQueueUtils.updateUsedCapacity(resourceCalculator,
         labelManager.getResourceByLabel(nodeLabel, Resources.none()),
-        Resources.none(), nodeLabel, this);
+        nodeLabel, this);
     if (null != parent) {
       parent.incUsedResource(nodeLabel, resourceToInc, null);
     }
@@ -948,7 +948,7 @@ public abstract class AbstractCSQueue implements CSQueue {
     queueUsage.decUsed(nodeLabel, resourceToDec);
     CSQueueUtils.updateUsedCapacity(resourceCalculator,
         labelManager.getResourceByLabel(nodeLabel, Resources.none()),
-        Resources.none(), nodeLabel, this);
+        nodeLabel, this);
     if (null != parent) {
       parent.decUsedResource(nodeLabel, resourceToDec, null);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7b8cd53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 81dec80..6daca51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -180,8 +180,8 @@ class CSQueueUtils {
    * used resource for all partitions of this queue.
    */
   public static void updateUsedCapacity(final ResourceCalculator rc,
-      final Resource totalPartitionResource, Resource clusterResource,
-      String nodePartition, AbstractCSQueue childQueue) {
+      final Resource totalPartitionResource, String nodePartition,
+      AbstractCSQueue childQueue) {
     QueueCapacities queueCapacities = childQueue.getQueueCapacities();
     CSQueueMetrics queueMetrics = childQueue.getMetrics();
     ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -287,11 +287,11 @@ class CSQueueUtils {
       for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
           queueResourceUsage.getNodePartitionsSet())) {
         updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
-            cluster, partition, childQueue);
+            partition, childQueue);
       }
     } else {
       updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
-          cluster, nodePartition, childQueue);
+          nodePartition, childQueue);
     }
 
     // Update queue metrics w.r.t node labels. In a generic way, we can

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7b8cd53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index a427fb1..c198d13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -739,8 +739,8 @@ public class ParentQueue extends AbstractCSQueue {
         child.getQueueResourceUsage().getUsed(nodePartition));
 
     // Get child's max resource
-    Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
-        nodePartition, minimumAllocation);
+    Resource childConfiguredMaxResource = child
+        .getEffectiveMaxCapacityDown(nodePartition, minimumAllocation);
 
     // Child's limit should be capped by child configured max resource
     childLimit =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7b8cd53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 23a371a..e91f734 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4307,7 +4307,7 @@ public class TestCapacityScheduler {
           null, null, NULL_UPDATE_REQUESTS);
       CapacityScheduler.schedule(cs);
     }
-    assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
+    assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
         cs.getQueue("p2").getUsedResources().getMemorySize());
 
     //Free a container from X1


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[40/50] hadoop git commit: HDFS-12833. Distcp : Update the usage of delete option for dependency with update and overwrite option. Contributed by usharani.

Posted by xy...@apache.org.
HDFS-12833. Distcp : Update the usage of delete option for dependency with update and overwrite option.  Contributed by usharani.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00129c53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00129c53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00129c53

Branch: refs/heads/HDFS-7240
Commit: 00129c5314dcd9bafa8138dbbcd51a173edbf098
Parents: 312ceeb
Author: Surendra Singh Lilhore <su...@apache.org>
Authored: Tue Dec 12 00:28:02 2017 +0530
Committer: Surendra Singh Lilhore <su...@apache.org>
Committed: Tue Dec 12 00:28:02 2017 +0530

----------------------------------------------------------------------
 .../src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java  | 2 +-
 hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/00129c53/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index e1c824e..faef7e6 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -73,7 +73,7 @@ public enum DistCpOptionSwitch {
    */
   DELETE_MISSING(DistCpConstants.CONF_LABEL_DELETE_MISSING,
       new Option("delete", false, "Delete from target, " +
-          "files missing in source")),
+          "files missing in source. Delete is applicable only with update or overwrite options")),
   /**
    * Number of threads for building source file listing (before map-reduce
    * phase, max one listStatus per thread at a time).

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00129c53/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
index 2cd01e2..8aa89f0 100644
--- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
+++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm
@@ -229,7 +229,7 @@ Flag              | Description                          | Notes
 `-filters` | The path to a file containing a list of pattern strings, one string per line, such that paths matching the pattern will be excluded from the copy. | Support regular expressions specified by java.util.regex.Pattern.
 `-filelimit <n>` | Limit the total number of files to be <= n | **Deprecated!** Ignored in the new DistCp.
 `-sizelimit <n>` | Limit the total size to be <= n bytes | **Deprecated!** Ignored in the new DistCp.
-`-delete` | Delete the files existing in the dst but not in src | The deletion is done by FS Shell. So the trash will be used, if it is enable.
+`-delete` | Delete the files existing in the dst but not in src | The deletion is done by FS Shell. So the trash will be used, if it is enable. Delete is applicable only with update or overwrite options.
 `-strategy {dynamic|uniformsize}` | Choose the copy-strategy to be used in DistCp. | By default, uniformsize is used. (i.e. Maps are balanced on the total size of files copied by each map. Similar to legacy.) If "dynamic" is specified, `DynamicInputFormat` is used instead. (This is described in the Architecture section, under InputFormats.)
 `-bandwidth` | Specify bandwidth per map, in MB/second. | Each map will be restricted to consume only the specified bandwidth. This is not always exact. The map throttles back its bandwidth consumption during a copy, such that the **net** bandwidth used tends towards the specified value.
 `-atomic {-tmp <tmp_dir>}` | Specify atomic commit, with optional tmp directory. | `-atomic` instructs DistCp to copy the source data to a temporary target location, and then move the temporary target to the final-location atomically. Data will either be available at final target in a complete and consistent form, or not at all. Optionally, `-tmp` may be used to specify the location of the tmp-target. If not specified, a default is chosen. **Note:** tmp_dir must be on the final target cluster.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[39/50] hadoop git commit: YARN-7632. Effective min and max resource need to be set for auto created leaf queues upon creation and capacity management. Contributed by Suma Shivaprasad.

Posted by xy...@apache.org.
YARN-7632. Effective min and max resource need to be set for auto created leaf queues upon creation and capacity management. Contributed by Suma Shivaprasad.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/312ceebd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/312ceebd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/312ceebd

Branch: refs/heads/HDFS-7240
Commit: 312ceebde8ef8881fc43d82a096fb852f833a206
Parents: a2edc4c
Author: Sunil G <su...@apache.org>
Authored: Mon Dec 11 19:20:02 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Mon Dec 11 19:20:02 2017 +0530

----------------------------------------------------------------------
 .../capacity/AutoCreatedLeafQueue.java          |  19 ++-
 .../TestAbsoluteResourceConfiguration.java      |   2 +-
 ...stCapacitySchedulerAutoCreatedQueueBase.java | 126 +++++++++++++++++--
 .../TestCapacitySchedulerAutoQueueCreation.java | 118 +++++++++++++++--
 .../scheduler/capacity/TestUtils.java           |   2 +-
 5 files changed, 238 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/312ceebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index 1d796ad..8b67087 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,14 +103,23 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue {
 
   private void mergeCapacities(QueueCapacities capacities) {
     for ( String nodeLabel : capacities.getExistingNodeLabels()) {
-      this.queueCapacities.setCapacity(nodeLabel,
+      queueCapacities.setCapacity(nodeLabel,
           capacities.getCapacity(nodeLabel));
-      this.queueCapacities.setAbsoluteCapacity(nodeLabel, capacities
+      queueCapacities.setAbsoluteCapacity(nodeLabel, capacities
           .getAbsoluteCapacity(nodeLabel));
-      this.queueCapacities.setMaximumCapacity(nodeLabel, capacities
+      queueCapacities.setMaximumCapacity(nodeLabel, capacities
           .getMaximumCapacity(nodeLabel));
-      this.queueCapacities.setAbsoluteMaximumCapacity(nodeLabel, capacities
+      queueCapacities.setAbsoluteMaximumCapacity(nodeLabel, capacities
           .getAbsoluteMaximumCapacity(nodeLabel));
+
+      Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+          csContext.getClusterResource());
+      getQueueResourceQuotas().setEffectiveMinResource(nodeLabel,
+          Resources.multiply(resourceByLabel,
+              queueCapacities.getAbsoluteCapacity(nodeLabel)));
+      getQueueResourceQuotas().setEffectiveMaxResource(nodeLabel,
+          Resources.multiply(resourceByLabel, queueCapacities
+              .getAbsoluteMaximumCapacity(nodeLabel)));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312ceebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 5c4a12e..298e1ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -513,7 +513,7 @@ public class TestAbsoluteResourceConfiguration {
   }
 
   @Test
-  public void testEffectiveResourceAfterIncreasinClusterResource()
+  public void testEffectiveResourceAfterIncreasingClusterResource()
       throws Exception {
     // create conf with basic queue configuration.
     CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312ceebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index 4e77339..d6282a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -17,17 +17,25 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import org.apache.commons.lang.math.RandomUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Event;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels
+    .NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement
     .ApplicationPlacementContext;
 import org.apache.hadoop.yarn.server.resourcemanager.placement
@@ -51,13 +59,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
     .SchedulerEvent;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -73,13 +85,14 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
     .capacity.CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class TestCapacitySchedulerAutoCreatedQueueBase {
 
   private static final Log LOG = LogFactory.getLog(
       TestCapacitySchedulerAutoCreatedQueueBase.class);
-  public final int GB = 1024;
+  public static final int GB = 1024;
   public final static ContainerUpdates NULL_UPDATE_REQUESTS =
       new ContainerUpdates();
 
@@ -107,6 +120,12 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
   public static final float C1_CAPACITY = 20f;
   public static final float C2_CAPACITY = 20f;
 
+  public static final int NODE_MEMORY = 16;
+
+  public static final int NODE1_VCORES = 16;
+  public static final int NODE2_VCORES = 32;
+  public static final int NODE3_VCORES = 48;
+
   public static final String USER = "user_";
   public static final String USER0 = USER + 0;
   public static final String USER1 = USER + 1;
@@ -120,6 +139,9 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
   public static final String NODEL_LABEL_SSD = "SSD";
 
   protected MockRM mockRM = null;
+  protected MockNM nm1 = null;
+  protected MockNM nm2 = null;
+  protected MockNM nm3 = null;
   protected CapacityScheduler cs;
   private final TestCapacityScheduler tcs = new TestCapacityScheduler();
   protected SpyDispatcher dispatcher;
@@ -163,16 +185,43 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
 
     setupQueueMappings(conf);
 
-    mockRM = new MockRM(conf);
-    cs = (CapacityScheduler) mockRM.getResourceScheduler();
-
     dispatcher = new SpyDispatcher();
     rmAppEventEventHandler = new SpyDispatcher.SpyRMAppEventHandler();
     dispatcher.register(RMAppEventType.class, rmAppEventEventHandler);
+
+    RMNodeLabelsManager mgr = setupNodeLabelManager(conf);
+
+    mockRM = new MockRM(conf) {
+      protected RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+
+    cs = (CapacityScheduler) mockRM.getResourceScheduler();
     cs.updatePlacementRules();
     mockRM.start();
-
     cs.start();
+
+    setupNodes(mockRM);
+  }
+
+  protected void setupNodes(MockRM newMockRM) throws Exception {
+    nm1 = // label = SSD
+        new MockNM("h1:1234", NODE_MEMORY * GB, NODE1_VCORES, newMockRM
+            .getResourceTrackerService());
+    nm1.registerNode();
+
+    nm2 = // label = GPU
+        new MockNM("h2:1234", NODE_MEMORY * GB, NODE2_VCORES, newMockRM
+            .getResourceTrackerService
+            ());
+    nm2.registerNode();
+
+    nm3 = // label = ""
+        new MockNM("h3:1234", NODE_MEMORY * GB, NODE3_VCORES, newMockRM
+            .getResourceTrackerService
+                ());
+    nm3.registerNode();
   }
 
   public static CapacitySchedulerConfiguration setupQueueMappings(
@@ -340,21 +389,23 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
     return queueMappings;
   }
 
-  protected MockRM setupSchedulerInstance() {
+  protected MockRM setupSchedulerInstance() throws Exception {
     CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
     setupQueueConfiguration(conf);
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
         ResourceScheduler.class);
 
-    List<String> queuePlacementRules = new ArrayList<String>();
-    queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
-    conf.setQueuePlacementRules(queuePlacementRules);
-
     setupQueueMappings(conf);
 
-    MockRM newMockRM = new MockRM(conf);
+    RMNodeLabelsManager mgr = setupNodeLabelManager(conf);
+    MockRM newMockRM = new MockRM(conf) {
+      protected RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
     newMockRM.start();
     ((CapacityScheduler) newMockRM.getResourceScheduler()).start();
+    setupNodes(newMockRM);
     return newMockRM;
   }
 
@@ -390,6 +441,21 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
     return parentQueue + DOT + leafQueue;
   }
 
+  protected RMNodeLabelsManager setupNodeLabelManager(
+      CapacitySchedulerConfiguration conf) throws IOException {
+    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+    mgr.init(conf);
+    mgr.addToCluserNodeLabelsWithDefaultExclusivity(
+        ImmutableSet.of(NODEL_LABEL_SSD, NODEL_LABEL_GPU));
+    mgr.addLabelsToNode(ImmutableMap
+        .of(NodeId.newInstance("h1", 0),
+            TestUtils.toSet(NODEL_LABEL_SSD)));
+    mgr.addLabelsToNode(ImmutableMap
+        .of(NodeId.newInstance("h2", 0),
+            TestUtils.toSet(NODEL_LABEL_GPU)));
+    return mgr;
+  }
+
   protected ApplicationAttemptId submitApp(CapacityScheduler newCS, String user,
       String queue, String parentQueue) {
     ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
@@ -460,8 +526,19 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
     AutoCreatedLeafQueue leafQueue =
         (AutoCreatedLeafQueue) capacityScheduler.getQueue(leafQueueName);
 
+    Map<String, QueueEntitlement> expectedEntitlements = new HashMap<>();
+    QueueCapacities cap = autoCreateEnabledParentQueue.getLeafQueueTemplate()
+        .getQueueCapacities();
+
     for (String label : accessibleNodeLabelsOnC) {
       validateCapacitiesByLabel(autoCreateEnabledParentQueue, leafQueue, label);
+
+      QueueEntitlement expectedEntitlement = new QueueEntitlement(
+          cap.getCapacity(label), cap.getMaximumCapacity(label));
+
+      expectedEntitlements.put(label, expectedEntitlement);
+
+      validateEffectiveMinResource(leafQueue, label, expectedEntitlements);
     }
 
     assertEquals(true, policy.isActive(leafQueue));
@@ -480,6 +557,28 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
         leafQueue.getQueueCapacities().getMaximumCapacity(label), EPSILON);
   }
 
+  protected void validateEffectiveMinResource(CSQueue leafQueue,
+      String label, Map<String, QueueEntitlement> expectedQueueEntitlements) {
+    ManagedParentQueue parentQueue = (ManagedParentQueue) leafQueue.getParent();
+
+    Resource resourceByLabel = mockRM.getRMContext().getNodeLabelManager().
+        getResourceByLabel(label, cs.getClusterResource());
+    Resource effMinCapacity = Resources.multiply(resourceByLabel,
+        expectedQueueEntitlements.get(label).getCapacity() * parentQueue
+            .getQueueCapacities().getAbsoluteCapacity(label));
+    assertEquals(effMinCapacity, Resources.multiply(resourceByLabel,
+        leafQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+    assertEquals(effMinCapacity, leafQueue.getEffectiveCapacity(label));
+
+    if (leafQueue.getQueueCapacities().getAbsoluteCapacity(label) > 0) {
+      assertTrue(Resources
+          .greaterThan(cs.getResourceCalculator(), cs.getClusterResource(),
+              effMinCapacity, Resources.none()));
+    } else{
+      assertTrue(Resources.equals(effMinCapacity, Resources.none()));
+    }
+  }
+
   protected void validateActivatedQueueEntitlement(CSQueue parentQueue,
       String leafQueueName, float expectedTotalChildQueueAbsCapacity,
       List<QueueManagementChange> queueManagementChanges)
@@ -552,6 +651,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
       QueueEntitlement expectedQueueEntitlement,
       final List<QueueManagementChange> queueEntitlementChanges) {
     boolean found = false;
+
+    Map<String, QueueEntitlement> expectedQueueEntitlements = new HashMap<>();
     for (QueueManagementChange entitlementChange : queueEntitlementChanges) {
       if (leafQueue.getQueueName().equals(
           entitlementChange.getQueue().getQueueName())) {
@@ -565,6 +666,9 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
               updatedQueueTemplate.getQueueCapacities()
                   .getMaximumCapacity(label));
           assertEquals(expectedQueueEntitlement, newEntitlement);
+          expectedQueueEntitlements.put(label, expectedQueueEntitlement);
+          validateEffectiveMinResource(leafQueue, label,
+              expectedQueueEntitlements);
         }
         found = true;
         break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312ceebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 049a932..a3b88c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -64,6 +65,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.security
 import org.apache.hadoop.yarn.server.resourcemanager.security
     .RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -91,12 +94,19 @@ public class TestCapacitySchedulerAutoQueueCreation
   private static final Log LOG = LogFactory.getLog(
       TestCapacitySchedulerAutoQueueCreation.class);
 
+  private static final Resource TEMPLATE_MAX_RES = Resource.newInstance(16 *
+          GB,
+      48);
+  private static final Resource TEMPLATE_MIN_RES = Resource.newInstance(1638,
+      4);
+
+
   @Test(timeout = 10000)
   public void testAutoCreateLeafQueueCreation() throws Exception {
 
     try {
       // submit an app
-      submitApp(cs, USER0, USER0, PARENT_QUEUE);
+      submitApp(mockRM, cs.getQueue(PARENT_QUEUE), USER0, USER0, 1, 1);
 
       // check preconditions
       List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
@@ -419,7 +429,7 @@ public class TestCapacitySchedulerAutoQueueCreation
 
   @Test
   public void testAutoCreationFailsWhenParentCapacityExceeded()
-      throws IOException, SchedulerDynamicEditException {
+      throws Exception {
     MockRM newMockRM = setupSchedulerInstance();
     CapacityScheduler newCS =
         (CapacityScheduler) newMockRM.getResourceScheduler();
@@ -468,11 +478,6 @@ public class TestCapacitySchedulerAutoQueueCreation
   public void testAutoCreatedQueueActivationDeactivation() throws Exception {
 
     try {
-      String host = "127.0.0.1";
-      RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
-          host);
-      cs.handle(new NodeAddedSchedulerEvent(node));
-
       CSQueue parentQueue = cs.getQueue(PARENT_QUEUE);
 
       //submit app1 as USER1
@@ -531,6 +536,100 @@ public class TestCapacitySchedulerAutoQueueCreation
   }
 
   @Test
+  public void testClusterResourceUpdationOnAutoCreatedLeafQueues() throws
+      Exception {
+
+    MockRM newMockRM = setupSchedulerInstance();
+    try {
+      CapacityScheduler newCS =
+          (CapacityScheduler) newMockRM.getResourceScheduler();
+
+      CSQueue parentQueue = newCS.getQueue(PARENT_QUEUE);
+
+      //submit app1 as USER1
+      submitApp(newMockRM, parentQueue, USER1, USER1, 1, 1);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER1, 0.1f);
+      CSQueue user1LeafQueue = newCS.getQueue(USER1);
+
+      //submit another app2 as USER2
+      submitApp(newMockRM, parentQueue, USER2, USER2, 2, 1);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER2, 0.2f);
+      CSQueue user2LeafQueue = newCS.getQueue(USER2);
+
+      //validate total activated abs capacity remains the same
+      GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
+          (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
+              parentQueue)
+              .getAutoCreatedQueueManagementPolicy();
+      assertEquals(autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), 0.2f, EPSILON);
+
+      //submit user_3 app. This cant be scheduled since there is no capacity
+      submitApp(newMockRM, parentQueue, USER3, USER3, 3, 1);
+      final CSQueue user3LeafQueue = newCS.getQueue(USER3);
+      validateCapacities((AutoCreatedLeafQueue) user3LeafQueue, 0.0f, 0.0f,
+          1.0f, 1.0f);
+
+      assertEquals(autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), 0.2f, EPSILON);
+
+      // add new NM.
+      newMockRM.registerNode("127.0.0.3:1234", 125 * GB, 20);
+
+      // There will be change in effective resource when nodes are added
+      // since we deal with percentages
+
+      Resource MAX_RES = Resources.addTo(TEMPLATE_MAX_RES,
+          Resources.createResource(125 * GB, 20));
+
+      Resource MIN_RES = Resources.createResource(14438, 6);
+
+      Assert.assertEquals("Effective Min resource for USER3 is not correct",
+          Resources.none(),
+          user3LeafQueue.getQueueResourceQuotas().getEffectiveMinResource());
+      Assert.assertEquals("Effective Max resource for USER3 is not correct",
+          MAX_RES,
+          user3LeafQueue.getQueueResourceQuotas().getEffectiveMaxResource());
+
+      Assert.assertEquals("Effective Min resource for USER2 is not correct",
+          MIN_RES,
+          user1LeafQueue.getQueueResourceQuotas().getEffectiveMinResource());
+      Assert.assertEquals("Effective Max resource for USER2 is not correct",
+          MAX_RES,
+          user1LeafQueue.getQueueResourceQuotas().getEffectiveMaxResource());
+
+      Assert.assertEquals("Effective Min resource for USER1 is not correct",
+          MIN_RES,
+          user2LeafQueue.getQueueResourceQuotas().getEffectiveMinResource());
+      Assert.assertEquals("Effective Max resource for USER1 is not correct",
+          MAX_RES,
+          user2LeafQueue.getQueueResourceQuotas().getEffectiveMaxResource());
+
+      // unregister one NM.
+      newMockRM.unRegisterNode(nm3);
+      Resource MIN_RES_UPDATED = Resources.createResource(12800, 2);
+      Resource MAX_RES_UPDATED = Resources.createResource(128000, 20);
+
+      // After loosing one NM, resources will reduce
+      Assert.assertEquals("Effective Min resource for USER2 is not correct",
+          MIN_RES_UPDATED,
+          user1LeafQueue.getQueueResourceQuotas().getEffectiveMinResource());
+      Assert.assertEquals("Effective Max resource for USER2 is not correct",
+          MAX_RES_UPDATED,
+          user2LeafQueue.getQueueResourceQuotas().getEffectiveMaxResource());
+
+    } finally {
+      cleanupQueue(USER1);
+      cleanupQueue(USER2);
+      cleanupQueue(USER3);
+      if (newMockRM != null) {
+        ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+        newMockRM.stop();
+      }
+    }
+  }
+
+  @Test
   public void testAutoCreatedQueueInheritsNodeLabels() throws Exception {
 
     try {
@@ -559,11 +658,6 @@ public class TestCapacitySchedulerAutoQueueCreation
           (CapacityScheduler) newMockRM.getResourceScheduler();
       CapacitySchedulerConfiguration conf = newCS.getConfiguration();
 
-      String host = "127.0.0.1";
-      RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
-          host);
-      newCS.handle(new NodeAddedSchedulerEvent(node));
-
       CSQueue parentQueue = newCS.getQueue(PARENT_QUEUE);
 
       //submit app1 as USER1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/312ceebd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 8036a40..e3326c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -309,7 +309,7 @@ public class TestUtils {
     
     return conf;
   }
-  
+
   public static Configuration getComplexConfigurationWithQueueLabels(
       Configuration config) {
     CapacitySchedulerConfiguration conf =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[12/50] hadoop git commit: YARN-7544. Use queue-path.capacity/maximum-capacity to specify absolute min/max resources. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7544. Use queue-path.capacity/maximum-capacity to specify absolute min/max resources. (Sunil G via wangda)

Change-Id: I685341be213eee500f51e02f01c91def89391c17


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1012b901
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1012b901
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1012b901

Branch: refs/heads/HDFS-7240
Commit: 1012b901c8eeeb46c7c792c94ed0befca1c860b4
Parents: b7b8cd5
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 23 19:53:26 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../CapacitySchedulerConfiguration.java         | 79 +++++++++++++++-----
 .../TestAbsoluteResourceConfiguration.java      | 30 ++++----
 2 files changed, 74 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1012b901/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index f81e097..e47c8d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -329,7 +329,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
   public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
 
-  public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+  public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$";
+
+  private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
 
   public enum AbsoluteResourceType {
     MEMORY, VCORES;
@@ -411,14 +413,29 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   }
   
   public float getNonLabeledQueueCapacity(String queue) {
-    float capacity = queue.equals("root") ? 100.0f : getFloat(
-        getQueuePrefix(queue) + CAPACITY, 0f);
-    if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
-      throw new IllegalArgumentException("Illegal " +
-      		"capacity of " + capacity + " for queue " + queue);
+    String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY);
+    boolean matcher = (configuredCapacity != null)
+        && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+    if (matcher) {
+      // Return capacity in percentage as 0 for non-root queues and 100 for
+      // root.From AbstractCSQueue, absolute resource will be parsed and
+      // updated. Once nodes are added/removed in cluster, capacity in
+      // percentage will also be re-calculated.
+      return queue.equals("root") ? 100.0f : 0f;
     }
-    LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue) + 
-        ", capacity=" + capacity);
+
+    float capacity = queue.equals("root")
+        ? 100.0f
+        : (configuredCapacity == null)
+            ? 0f
+            : Float.parseFloat(configuredCapacity);
+    if (capacity < MINIMUM_CAPACITY_VALUE
+        || capacity > MAXIMUM_CAPACITY_VALUE) {
+      throw new IllegalArgumentException(
+          "Illegal " + "capacity of " + capacity + " for queue " + queue);
+    }
+    LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue)
+        + ", capacity=" + capacity);
     return capacity;
   }
   
@@ -433,10 +450,23 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   }
 
   public float getNonLabeledQueueMaximumCapacity(String queue) {
-    float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
-        MAXIMUM_CAPACITY_VALUE);
-    maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? 
-        MAXIMUM_CAPACITY_VALUE : maxCapacity;
+    String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY);
+    boolean matcher = (configuredCapacity != null)
+        && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+    if (matcher) {
+      // Return capacity in percentage as 0 for non-root queues and 100 for
+      // root.From AbstractCSQueue, absolute resource will be parsed and
+      // updated. Once nodes are added/removed in cluster, capacity in
+      // percentage will also be re-calculated.
+      return 100.0f;
+    }
+
+    float maxCapacity = (configuredCapacity == null)
+        ? MAXIMUM_CAPACITY_VALUE
+        : Float.parseFloat(configuredCapacity);
+    maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE)
+        ? MAXIMUM_CAPACITY_VALUE
+        : maxCapacity;
     return maxCapacity;
   }
   
@@ -590,6 +620,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   private float internalGetLabeledQueueCapacity(String queue, String label, String suffix,
       float defaultValue) {
     String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
+    boolean matcher = (capacityPropertyName != null)
+        && RESOURCE_PATTERN.matcher(capacityPropertyName).find();
+    if (matcher) {
+      // Return capacity in percentage as 0 for non-root queues and 100 for
+      // root.From AbstractCSQueue, absolute resource will be parsed and
+      // updated. Once nodes are added/removed in cluster, capacity in
+      // percentage will also be re-calculated.
+      return defaultValue;
+    }
+
     float capacity = getFloat(capacityPropertyName, defaultValue);
     if (capacity < MINIMUM_CAPACITY_VALUE
         || capacity > MAXIMUM_CAPACITY_VALUE) {
@@ -1722,7 +1762,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public Resource getMinimumResourceRequirement(String label, String queue,
       Set<String> resourceTypes) {
     return internalGetLabeledResourceRequirementForQueue(queue, label,
-        resourceTypes, MINIMUM_RESOURCE);
+        resourceTypes, CAPACITY);
   }
 
   /**
@@ -1739,19 +1779,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public Resource getMaximumResourceRequirement(String label, String queue,
       Set<String> resourceTypes) {
     return internalGetLabeledResourceRequirementForQueue(queue, label,
-        resourceTypes, MAXIMUM_RESOURCE);
+        resourceTypes, MAXIMUM_CAPACITY);
   }
 
   @VisibleForTesting
   public void setMinimumResourceRequirement(String label, String queue,
       Resource resource) {
-    updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+    updateMinMaxResourceToConf(label, queue, resource, CAPACITY);
   }
 
   @VisibleForTesting
   public void setMaximumResourceRequirement(String label, String queue,
       Resource resource) {
-    updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+    updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_CAPACITY);
   }
 
   private void updateMinMaxResourceToConf(String label, String queue,
@@ -1786,8 +1826,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
     // Define resource here.
     Resource resource = Resource.newInstance(0l, 0);
-    Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
-        .matcher(resourceString);
+    Matcher matcher = RESOURCE_PATTERN.matcher(resourceString);
+
     /*
      * Absolute resource configuration for a queue will be grouped by "[]".
      * Syntax of absolute resource config could be like below
@@ -1795,11 +1835,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
      */
     if (matcher.find()) {
       // Get the sub-group.
-      String subGroup = matcher.group(1);
+      String subGroup = matcher.group(0);
       if (subGroup.trim().isEmpty()) {
         return Resources.none();
       }
 
+      subGroup = subGroup.substring(1, subGroup.length() - 1);
       for (String kvPair : subGroup.trim().split(",")) {
         String[] splits = kvPair.split("=");
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1012b901/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 5a66281..80d16bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -233,7 +232,12 @@ public class TestAbsoluteResourceConfiguration {
   @Test
   public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
     /**
-     * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+     * Queue structure is as follows.
+     *    root
+     *   / | \
+     *   a b c
+     *   / \ |
+     *  a1 a2 b1
      *
      * Test below cases 1) Configure percentage based capacity and absolute
      * resource together. 2) As per above tree structure, ensure all values
@@ -258,21 +262,15 @@ public class TestAbsoluteResourceConfiguration {
     // Get queue object to verify min/max resource configuration.
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
 
-    // 1. Create a new config with capcity and min/max together. Ensure an
-    // exception is thrown.
+    // 1. Create a new config with min/max.
     CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
         true);
     setupMinMaxResourceConfiguration(csConf1);
 
     try {
       cs.reinitialize(csConf1, rm.getRMContext());
-      Assert.fail();
     } catch (IOException e) {
-      Assert.assertTrue(e instanceof IOException);
-      Assert.assertEquals(
-          "Failed to re-init queues : Queue 'queueA' should use either"
-              + " percentage based capacity configuration or absolute resource.",
-          e.getMessage());
+      Assert.fail();
     }
     rm.stop();
 
@@ -368,7 +366,12 @@ public class TestAbsoluteResourceConfiguration {
   @Test
   public void testComplexValidateAbsoluteResourceConfig() throws Exception {
     /**
-     * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+     * Queue structure is as follows.
+     *   root
+     *  / | \
+     *  a b c
+     * / \ |
+     * a1 a2 b1
      *
      * Test below cases: 1) Parent and its child queues must use either
      * percentage based or absolute resource configuration. 2) Parent's min
@@ -396,11 +399,6 @@ public class TestAbsoluteResourceConfiguration {
     csConf.setCapacity(QUEUEB_FULL, 25f);
     csConf.setCapacity(QUEUEC_FULL, 25f);
 
-    // Also unset resource based config.
-    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
-    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
-    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
-
     // Get queue object to verify min/max resource configuration.
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
     try {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[10/50] hadoop git commit: HDFS-11915. Sync rbw dir on the first hsync() to avoid file lost on power failure. Contributed by Vinayakumar B.

Posted by xy...@apache.org.
HDFS-11915. Sync rbw dir on the first hsync() to avoid file lost on power failure. Contributed by Vinayakumar B.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6c31a3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6c31a3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6c31a3e

Branch: refs/heads/HDFS-7240
Commit: d6c31a3e6b60c4b8af9ae4661f16614805654e59
Parents: acb9290
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Dec 7 14:50:30 2017 -0800
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Dec 7 14:50:30 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java   | 5 +++++
 1 file changed, 5 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6c31a3e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index c052d52..afb9e2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -126,6 +126,7 @@ class BlockReceiver implements Closeable {
 
   private boolean syncOnClose;
   private volatile boolean dirSyncOnFinalize;
+  private boolean dirSyncOnHSyncDone = false;
   private long restartBudget;
   /** the reference of the volume where the block receiver writes to */
   private ReplicaHandler replicaHandler;
@@ -424,6 +425,10 @@ class BlockReceiver implements Closeable {
       }
       flushTotalNanos += flushEndNanos - flushStartNanos;
     }
+    if (isSync && !dirSyncOnHSyncDone && replicaInfo instanceof LocalReplica) {
+      ((LocalReplica) replicaInfo).fsyncDirectory();
+      dirSyncOnHSyncDone = true;
+    }
     if (checksumOut != null || streams.getDataOut() != null) {
       datanode.metrics.addFlushNanos(flushTotalNanos);
       if (isSync) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[33/50] hadoop git commit: YARN-7420. YARN UI changes to depict auto created queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
YARN-7420. YARN UI changes to depict auto created queues. (Suma Shivaprasad via wangda)

Change-Id: I8039d3772a191ddede132cd1f8b08a8ca2e275b7


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f548bfff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f548bfff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f548bfff

Branch: refs/heads/HDFS-7240
Commit: f548bfffbdcd426811352d6920ee5fe50cd0182c
Parents: b38643c
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:10:47 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:10:47 2017 -0800

----------------------------------------------------------------------
 .../resourcemanager/webapp/CapacitySchedulerPage.java | 14 ++++++++++++--
 .../webapp/dao/CapacitySchedulerLeafQueueInfo.java    | 11 +++++++++++
 .../webapp/dao/CapacitySchedulerQueueInfo.java        |  4 ++++
 3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f548bfff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 4933d34..de85590 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -66,6 +66,7 @@ class CapacitySchedulerPage extends RmView {
   static final String Q_END = "left:101%";
   static final String Q_GIVEN =
       "left:0%;background:none;border:1px dashed #BFBFBF";
+  static final String Q_AUTO_CREATED = "background:#F4F0CB";
   static final String Q_OVER = "background:#FFA333";
   static final String Q_UNDER = "background:#5BD75B";
   static final String ACTIVE_USER = "background:#FFFF00"; // Yellow highlight
@@ -299,9 +300,16 @@ class CapacitySchedulerPage extends RmView {
         absMaxCap = partitionQueueCapsInfo.getAbsoluteMaxCapacity() / 100;
         absUsedCap = partitionQueueCapsInfo.getAbsoluteUsedCapacity() / 100;
 
+        boolean isAutoCreatedLeafQueue = info.isLeafQueue() ?
+            ((CapacitySchedulerLeafQueueInfo) info).isAutoCreatedLeafQueue()
+            : false;
+
+        String Q_WIDTH = width(absMaxCap * Q_MAX_WIDTH);
         LI<UL<Hamlet>> li = ul.
           li().
-            a(_Q).$style(width(absMaxCap * Q_MAX_WIDTH)).
+            a(_Q).$style(isAutoCreatedLeafQueue? join( Q_AUTO_CREATED, ";",
+            Q_WIDTH)
+            :  Q_WIDTH).
               $title(join("Absolute Capacity:", percent(absCap))).
               span().$style(join(Q_GIVEN, ";font-size:1px;", width(absCap/absMaxCap))).
             __('.').__().
@@ -313,7 +321,7 @@ class CapacitySchedulerPage extends RmView {
             __(join(percent(used), " used")).__();
 
         csqinfo.qinfo = info;
-        if (info.getQueues() == null) {
+        if (info.isLeafQueue()) {
           li.ul("#lq").li().__(LeafQueueInfoBlock.class).__().__();
           li.ul("#lq").li().__(QueueUsersInfoBlock.class).__().__();
         } else {
@@ -422,6 +430,8 @@ class CapacitySchedulerPage extends RmView {
               __("Max Capacity").__().
             span().$class("qlegend ui-corner-all").$style(ACTIVE_USER).
             __("Users Requesting Resources").__().
+            span().$class("qlegend ui-corner-all").$style(Q_AUTO_CREATED).
+            __("Auto Created Queues").__().
           __();
 
         float used = 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f548bfff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 343c78e..b5f4e79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -27,6 +27,8 @@ import javax.xml.bind.annotation.XmlTransient;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@@ -49,6 +51,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   protected boolean preemptionDisabled;
   protected String defaultNodeLabelExpression;
   protected int defaultPriority;
+  protected boolean isAutoCreatedLeafQueue;
 
   @XmlTransient
   protected String orderingPolicyInfo;
@@ -82,6 +85,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
           .getPartitionResourceUsageInfo(RMNodeLabelsManager.NO_LABEL)
           .getAMLimit();
     }
+
+    if ( q instanceof AutoCreatedLeafQueue) {
+      isAutoCreatedLeafQueue = true;
+    }
   }
 
   @Override
@@ -155,4 +162,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   public int getDefaultApplicationPriority() {
     return defaultPriority;
   }
+
+  public boolean isAutoCreatedLeafQueue() {
+    return isAutoCreatedLeafQueue;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f548bfff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index d4de9ae..fd256b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -219,4 +219,8 @@ public class CapacitySchedulerQueueInfo {
   public ResourceInfo getMaxEffectiveCapacity(){
     return maxEffectiveCapacity;
   }
+
+  public boolean isLeafQueue() {
+    return getQueues() == null;
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[08/50] hadoop git commit: HDFS-12840. Creating a file with non-default EC policy in a EC zone is not correctly serialized in the editlog. Contributed by Lei (Eddy) Xu.

Posted by xy...@apache.org.
HDFS-12840. Creating a file with non-default EC policy in a EC zone is not correctly serialized in the editlog. Contributed by Lei (Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67662e2a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67662e2a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67662e2a

Branch: refs/heads/HDFS-7240
Commit: 67662e2ac9e68f32b725c8118cf2be79a662fca5
Parents: 67b2661
Author: Lei Xu <le...@apache.org>
Authored: Thu Dec 7 11:15:40 2017 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Thu Dec 7 11:15:40 2017 -0800

----------------------------------------------------------------------
 .../io/erasurecode/ErasureCodeConstants.java    |   2 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  21 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |   3 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   2 +-
 .../hdfs/server/namenode/FSEditLogOp.java       |  26 +
 .../hadoop/hdfs/server/namenode/INodeFile.java  |  14 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |  23 +
 .../TestDistributedFileSystemWithECFile.java    |  55 ++
 .../namenode/OfflineEditsViewerHelper.java      |   4 +-
 .../server/namenode/TestNamenodeRetryCache.java |  14 +-
 .../namenode/ha/TestRetryCacheWithHA.java       |  13 +-
 .../hadoop-hdfs/src/test/resources/editsStored  | Bin 6753 -> 7909 bytes
 .../src/test/resources/editsStored.xml          | 536 +++++++++++++------
 13 files changed, 531 insertions(+), 182 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index 73b8f56..2eac016 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -52,6 +52,6 @@ public final class ErasureCodeConstants {
 
   public static final byte MAX_POLICY_ID = Byte.MAX_VALUE;
   public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
-  public static final byte REPLICATION_POLICY_ID = (byte) 63;
+  public static final byte REPLICATION_POLICY_ID = (byte) 0;
   public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index b202212..c4041a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.net.NodeBase;
 import org.apache.hadoop.util.ChunkedArrayList;
@@ -415,22 +416,28 @@ class FSDirWriteFileOp {
       PermissionStatus permissions, List<AclEntry> aclEntries,
       List<XAttr> xAttrs, short replication, long modificationTime, long atime,
       long preferredBlockSize, boolean underConstruction, String clientName,
-      String clientMachine, byte storagePolicyId) {
+      String clientMachine, byte storagePolicyId, byte ecPolicyID) {
     final INodeFile newNode;
     Preconditions.checkNotNull(existing);
     assert fsd.hasWriteLock();
     try {
       // check if the file has an EC policy
-      boolean isStriped = false;
-      ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
-          unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing);
-      if (ecPolicy != null) {
-        isStriped = true;
+      boolean isStriped =
+          ecPolicyID != ErasureCodeConstants.REPLICATION_POLICY_ID;
+      ErasureCodingPolicy ecPolicy = null;
+      if (isStriped) {
+        ecPolicy = fsd.getFSNamesystem().getErasureCodingPolicyManager()
+          .getByID(ecPolicyID);
+        if (ecPolicy == null) {
+          throw new IOException(String.format(
+              "Cannot find erasure coding policy for new file %s/%s, " +
+                  "ecPolicyID=%d",
+              existing.getPath(), Arrays.toString(localName), ecPolicyID));
+        }
       }
       final BlockType blockType = isStriped ?
           BlockType.STRIPED : BlockType.CONTIGUOUS;
       final Short replicationFactor = (!isStriped ? replication : null);
-      final Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null);
       if (underConstruction) {
         newNode = newINodeFile(id, permissions, modificationTime,
             modificationTime, replicationFactor, ecPolicyID, preferredBlockSize,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 72e00ee..d6fb212 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -815,7 +815,8 @@ public class FSEditLog implements LogsPurgeable {
       .setClientMachine(
           newNode.getFileUnderConstructionFeature().getClientMachine())
       .setOverwrite(overwrite)
-      .setStoragePolicyId(newNode.getLocalStoragePolicyID());
+      .setStoragePolicyId(newNode.getLocalStoragePolicyID())
+      .setErasureCodingPolicyId(newNode.getErasureCodingPolicyID());
 
     AclFeature f = newNode.getAclFeature();
     if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index a21b8ea..b0fe60a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -385,7 +385,7 @@ public class FSEditLogLoader {
             addCloseOp.xAttrs, replication, addCloseOp.mtime,
             addCloseOp.atime, addCloseOp.blockSize, true,
             addCloseOp.clientName, addCloseOp.clientMachine,
-            addCloseOp.storagePolicyId);
+            addCloseOp.storagePolicyId, addCloseOp.erasureCodingPolicyId);
         assert newFile != null;
         iip = INodesInPath.replace(iip, iip.length() - 1, newFile);
         fsNamesys.leaseManager.addLease(addCloseOp.clientName, newFile.getId());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 2dc9d33..c0daaf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -127,6 +127,7 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableFactories;
 import org.apache.hadoop.io.WritableFactory;
 import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.ipc.ClientId;
 import org.apache.hadoop.ipc.RpcConstants;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
@@ -425,10 +426,12 @@ public abstract class FSEditLogOp {
     String clientMachine;
     boolean overwrite;
     byte storagePolicyId;
+    byte erasureCodingPolicyId;
     
     private AddCloseOp(FSEditLogOpCodes opCode) {
       super(opCode);
       storagePolicyId = HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
+      erasureCodingPolicyId = ErasureCodeConstants.REPLICATION_POLICY_ID;
       assert(opCode == OP_ADD || opCode == OP_CLOSE || opCode == OP_APPEND);
     }
 
@@ -449,6 +452,7 @@ public abstract class FSEditLogOp {
       clientMachine = null;
       overwrite = false;
       storagePolicyId = 0;
+      erasureCodingPolicyId = ErasureCodeConstants.REPLICATION_POLICY_ID;
     }
 
     <T extends AddCloseOp> T setInodeId(long inodeId) {
@@ -535,6 +539,11 @@ public abstract class FSEditLogOp {
       return (T)this;
     }
 
+    <T extends AddCloseOp> T setErasureCodingPolicyId(byte ecPolicyId) {
+      this.erasureCodingPolicyId = ecPolicyId;
+      return (T)this;
+    }
+
     @Override
     public void writeFields(DataOutputStream out) throws IOException {
       FSImageSerialization.writeLong(inodeId, out);
@@ -555,6 +564,7 @@ public abstract class FSEditLogOp {
         FSImageSerialization.writeString(clientMachine,out);
         FSImageSerialization.writeBoolean(overwrite, out);
         FSImageSerialization.writeByte(storagePolicyId, out);
+        FSImageSerialization.writeByte(erasureCodingPolicyId, out);
         // write clientId and callId
         writeRpcIds(rpcClientId, rpcCallId, out);
       }
@@ -633,6 +643,14 @@ public abstract class FSEditLogOp {
           this.storagePolicyId =
               HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
         }
+
+        if (NameNodeLayoutVersion.supports(
+            NameNodeLayoutVersion.Feature.ERASURE_CODING, logVersion)) {
+          this.erasureCodingPolicyId = FSImageSerialization.readByte(in);
+        } else {
+          this.erasureCodingPolicyId =
+              ErasureCodeConstants.REPLICATION_POLICY_ID;
+        }
         // read clientId and callId
         readRpcIds(in, logVersion);
       } else {
@@ -695,6 +713,8 @@ public abstract class FSEditLogOp {
       }
       builder.append(", storagePolicyId=");
       builder.append(storagePolicyId);
+      builder.append(", erasureCodingPolicyId=");
+      builder.append(erasureCodingPolicyId);
       builder.append(", opCode=");
       builder.append(opCode);
       builder.append(", txid=");
@@ -730,6 +750,8 @@ public abstract class FSEditLogOp {
         if (aclEntries != null) {
           appendAclEntriesToXml(contentHandler, aclEntries);
         }
+        XMLUtils.addSaxString(contentHandler, "ERASURE_CODING_POLICY_ID",
+            Byte.toString(erasureCodingPolicyId));
         appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
       }
     }
@@ -758,6 +780,10 @@ public abstract class FSEditLogOp {
       }
       this.permissions = permissionStatusFromXml(st);
       aclEntries = readAclEntriesFromXml(st);
+      if (st.hasChildren("ERASURE_CODING_POLICY_ID")) {
+        this.erasureCodingPolicyId = Byte.parseByte(st.getValue(
+            "ERASURE_CODING_POLICY_ID"));
+      }
       readRpcIdsFromXml(st);
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index fcdb064..3f2fb33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.util.LongBitFormat;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.util.StringUtils;
 
 /** I-node for closed file. */
@@ -200,9 +201,10 @@ public class INodeFile extends INodeWithAdditionalFields
         // as the PolicyID can never be in negative.
         layoutRedundancy |= erasureCodingPolicyID;
       } else {
-        Preconditions.checkArgument(replication != null &&
-            erasureCodingPolicyID == null);
-        Preconditions.checkArgument(replication >= 0 &&
+        Preconditions.checkArgument(erasureCodingPolicyID == null ||
+                erasureCodingPolicyID ==
+                    ErasureCodeConstants.REPLICATION_POLICY_ID);
+        Preconditions.checkArgument(replication != null && replication >= 0 &&
             replication <= MAX_REDUNDANCY,
             "Invalid replication value " + replication);
         layoutRedundancy |= replication;
@@ -588,10 +590,8 @@ public class INodeFile extends INodeWithAdditionalFields
     setStoragePolicyID(storagePolicyId);
   }
 
-
   /**
-   * @return The ID of the erasure coding policy on the file. -1 represents no
-   *          EC policy.
+   * @return The ID of the erasure coding policy on the file.
    */
   @VisibleForTesting
   @Override
@@ -599,7 +599,7 @@ public class INodeFile extends INodeWithAdditionalFields
     if (isStriped()) {
       return HeaderFormat.getECPolicyID(header);
     }
-    return -1;
+    return ErasureCodeConstants.REPLICATION_POLICY_ID;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1048b0a..1411a7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1510,6 +1510,29 @@ public class DFSTestUtil {
     // OP_REMOVE_ERASURE_CODING_POLICY
     filesystem.removeErasureCodingPolicy(newPolicy1.getName());
     filesystem.removeErasureCodingPolicy(newPolicy2.getName());
+
+    // OP_ADD on erasure coding directory
+    Path ecDir = new Path("/ec");
+    filesystem.mkdirs(ecDir);
+    final ErasureCodingPolicy defaultEcPolicy =
+        SystemErasureCodingPolicies.getByID(
+            SystemErasureCodingPolicies.RS_6_3_POLICY_ID);
+    final ErasureCodingPolicy ecPolicyRS32 =
+        SystemErasureCodingPolicies.getByID(
+            SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
+    filesystem.enableErasureCodingPolicy(ecPolicyRS32.getName());
+    filesystem.enableErasureCodingPolicy(defaultEcPolicy.getName());
+    filesystem.setErasureCodingPolicy(ecDir, defaultEcPolicy.getName());
+
+    try (FSDataOutputStream out = filesystem.createFile(
+        new Path(ecDir, "replicated")).replicate().build()) {
+      out.write("replicated".getBytes());
+    }
+
+    try (FSDataOutputStream out = filesystem.createFile(
+        new Path(ecDir, "RS-3-2")).ecPolicyName(ecPolicyRS32.getName()).build()) {
+      out.write("RS-3-2".getBytes());
+    }
   }
 
   public static void abortStream(DFSOutputStream out) throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
index 14a2ec4..0a3010f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
@@ -19,11 +19,13 @@ package org.apache.hadoop.hdfs;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.LocatedFileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -32,6 +34,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -194,4 +198,55 @@ public class TestDistributedFileSystemWithECFile {
     assertTrue(lastBlock.getOffset() == blockGroupSize);
     assertTrue(lastBlock.getLength() == lastBlockSize);
   }
+
+  @Test(timeout=60000)
+  public void testReplayEditLogsForReplicatedFile() throws Exception {
+    cluster.shutdown();
+
+    ErasureCodingPolicy rs63 = SystemErasureCodingPolicies.getByID(
+        SystemErasureCodingPolicies.RS_6_3_POLICY_ID
+    );
+    ErasureCodingPolicy rs32 = SystemErasureCodingPolicies.getByID(
+        SystemErasureCodingPolicies.RS_3_2_POLICY_ID
+    );
+    // Test RS(6,3) as default policy
+    int numDataNodes = rs63.getNumDataUnits() + rs63.getNumParityUnits();
+    cluster = new MiniDFSCluster.Builder(conf)
+        .nnTopology(MiniDFSNNTopology.simpleHATopology())
+        .numDataNodes(numDataNodes)
+        .build();
+
+    cluster.transitionToActive(0);
+    fs = cluster.getFileSystem(0);
+    fs.enableErasureCodingPolicy(rs63.getName());
+    fs.enableErasureCodingPolicy(rs32.getName());
+
+    Path dir = new Path("/ec");
+    fs.mkdirs(dir);
+    fs.setErasureCodingPolicy(dir, rs63.getName());
+
+    // Create an erasure coded file with the default policy.
+    Path ecFile = new Path(dir, "ecFile");
+    createFile(ecFile.toString(), 10);
+    // Create a replicated file.
+    Path replicatedFile = new Path(dir, "replicated");
+    try (FSDataOutputStream out = fs.createFile(replicatedFile)
+      .replicate().build()) {
+      out.write(123);
+    }
+    // Create an EC file with a different policy.
+    Path ecFile2 = new Path(dir, "RS-3-2");
+    try (FSDataOutputStream out = fs.createFile(ecFile2)
+         .ecPolicyName(rs32.getName()).build()) {
+      out.write(456);
+    }
+
+    cluster.transitionToStandby(0);
+    cluster.transitionToActive(1);
+
+    fs = cluster.getFileSystem(1);
+    assertNull(fs.getErasureCodingPolicy(replicatedFile));
+    assertEquals(rs63, fs.getErasureCodingPolicy(ecFile));
+    assertEquals(rs32, fs.getErasureCodingPolicy(ecFile2));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
index b5ba108..d637af5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
@@ -98,8 +98,10 @@ public class OfflineEditsViewerHelper {
     config.setBoolean(
         DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
     config.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+    final int numDataNodes = 9;
     cluster =
-      new MiniDFSCluster.Builder(config).manageNameDfsDirs(false).build();
+      new MiniDFSCluster.Builder(config).manageNameDfsDirs(false)
+          .numDataNodes(numDataNodes).build();
     cluster.waitClusterUp();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
index 42ff698..0995f13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java
@@ -43,9 +43,11 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.MiniDFSNNTopology;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.ipc.ClientId;
@@ -79,6 +81,11 @@ import org.junit.Test;
 public class TestNamenodeRetryCache {
   private static final byte[] CLIENT_ID = ClientId.getClientId();
   private static MiniDFSCluster cluster;
+  private static ErasureCodingPolicy defaultEcPolicy =
+      SystemErasureCodingPolicies.getByID(
+          SystemErasureCodingPolicies.RS_6_3_POLICY_ID);
+  private static int numDataNodes = defaultEcPolicy.getNumDataUnits() +
+      defaultEcPolicy.getNumParityUnits() + 1;
   private static NamenodeProtocols nnRpc;
   private static final FsPermission perm = FsPermission.getDefault();
   private static DistributedFileSystem filesystem;
@@ -93,7 +100,8 @@ public class TestNamenodeRetryCache {
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BlockSize);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY, true);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
-    cluster = new MiniDFSCluster.Builder(conf).build();
+    cluster = new MiniDFSCluster.Builder(conf)
+        .numDataNodes(numDataNodes).build();
     cluster.waitActive();
     nnRpc = cluster.getNameNode().getRpcServer();
     filesystem = cluster.getFileSystem();
@@ -436,7 +444,7 @@ public class TestNamenodeRetryCache {
 
     LightWeightCache<CacheEntry, CacheEntry> cacheSet = 
         (LightWeightCache<CacheEntry, CacheEntry>) namesystem.getRetryCache().getCacheSet();
-    assertEquals("Retry cache size is wrong", 34, cacheSet.size());
+    assertEquals("Retry cache size is wrong", 39, cacheSet.size());
     
     Map<CacheEntry, CacheEntry> oldEntries = 
         new HashMap<CacheEntry, CacheEntry>();
@@ -455,7 +463,7 @@ public class TestNamenodeRetryCache {
     assertTrue(namesystem.hasRetryCache());
     cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) namesystem
         .getRetryCache().getCacheSet();
-    assertEquals("Retry cache size is wrong", 34, cacheSet.size());
+    assertEquals("Retry cache size is wrong", 39, cacheSet.size());
     iter = cacheSet.iterator();
     while (iter.hasNext()) {
       CacheEntry entry = iter.next();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index 1d114d6..43aaa92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -67,11 +67,13 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -93,7 +95,12 @@ public class TestRetryCacheWithHA {
   private static final Log LOG = LogFactory.getLog(TestRetryCacheWithHA.class);
   
   private static final int BlockSize = 1024;
-  private static final short DataNodes = 3;
+  private static ErasureCodingPolicy defaultEcPolicy =
+      SystemErasureCodingPolicies.getByID(
+          SystemErasureCodingPolicies.RS_6_3_POLICY_ID);
+  private static final short DataNodes = (short)(
+      defaultEcPolicy.getNumDataUnits() +
+      defaultEcPolicy.getNumParityUnits() + 1);
   private static final int CHECKTIMES = 10;
   private static final int ResponseSize = 3;
   
@@ -166,7 +173,7 @@ public class TestRetryCacheWithHA {
     FSNamesystem fsn0 = cluster.getNamesystem(0);
     LightWeightCache<CacheEntry, CacheEntry> cacheSet = 
         (LightWeightCache<CacheEntry, CacheEntry>) fsn0.getRetryCache().getCacheSet();
-    assertEquals("Retry cache size is wrong", 34, cacheSet.size());
+    assertEquals("Retry cache size is wrong", 39, cacheSet.size());
     
     Map<CacheEntry, CacheEntry> oldEntries = 
         new HashMap<CacheEntry, CacheEntry>();
@@ -187,7 +194,7 @@ public class TestRetryCacheWithHA {
     FSNamesystem fsn1 = cluster.getNamesystem(1);
     cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) fsn1
         .getRetryCache().getCacheSet();
-    assertEquals("Retry cache size is wrong", 34, cacheSet.size());
+    assertEquals("Retry cache size is wrong", 39, cacheSet.size());
     iter = cacheSet.iterator();
     while (iter.hasNext()) {
       CacheEntry entry = iter.next();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index 3f2817a..a0ae78e 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67662e2a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 2a57c73..7e1881c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -13,8 +13,8 @@
       <TXID>2</TXID>
       <DELEGATION_KEY>
         <KEY_ID>1</KEY_ID>
-        <EXPIRY_DATE>1512000829976</EXPIRY_DATE>
-        <KEY>e7457bcc6ab95a84</KEY>
+        <EXPIRY_DATE>1513298395825</EXPIRY_DATE>
+        <KEY>ddb3d2c37b57926a</KEY>
       </DELEGATION_KEY>
     </DATA>
   </RECORD>
@@ -24,8 +24,8 @@
       <TXID>3</TXID>
       <DELEGATION_KEY>
         <KEY_ID>2</KEY_ID>
-        <EXPIRY_DATE>1512000829980</EXPIRY_DATE>
-        <KEY>07cc38caf6c47bb4</KEY>
+        <EXPIRY_DATE>1513298395827</EXPIRY_DATE>
+        <KEY>57acfb80c8b539fa</KEY>
       </DELEGATION_KEY>
     </DATA>
   </RECORD>
@@ -36,11 +36,11 @@
       <LENGTH>0</LENGTH>
       <INODEID>16386</INODEID>
       <PATH>/file_create</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632199</MTIME>
-      <ATIME>1511309632199</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197452</MTIME>
+      <ATIME>1512607197452</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -48,8 +48,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>5</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>35</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -59,9 +60,9 @@
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632248</MTIME>
-      <ATIME>1511309632199</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197500</MTIME>
+      <ATIME>1512607197452</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -78,11 +79,11 @@
     <DATA>
       <TXID>6</TXID>
       <PATH>/file_create</PATH>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <NEWBLOCK>false</NEWBLOCK>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>7</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>37</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -92,9 +93,9 @@
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632263</MTIME>
-      <ATIME>1511309632199</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197516</MTIME>
+      <ATIME>1512607197452</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -114,10 +115,10 @@
       <INODEID>16387</INODEID>
       <PATH>/update_blocks</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632266</MTIME>
-      <ATIME>1511309632266</ATIME>
+      <MTIME>1512607197519</MTIME>
+      <ATIME>1512607197519</ATIME>
       <BLOCKSIZE>4096</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -125,8 +126,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>9</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>39</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -188,8 +190,8 @@
       <INODEID>0</INODEID>
       <PATH>/update_blocks</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632454</MTIME>
-      <ATIME>1511309632266</ATIME>
+      <MTIME>1512607197657</MTIME>
+      <ATIME>1512607197519</ATIME>
       <BLOCKSIZE>4096</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -216,9 +218,9 @@
       <LENGTH>0</LENGTH>
       <SRC>/file_create</SRC>
       <DST>/file_moved</DST>
-      <TIMESTAMP>1511309632467</TIMESTAMP>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>17</RPC_CALLID>
+      <TIMESTAMP>1512607197671</TIMESTAMP>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>47</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -227,9 +229,9 @@
       <TXID>17</TXID>
       <LENGTH>0</LENGTH>
       <PATH>/file_moved</PATH>
-      <TIMESTAMP>1511309632480</TIMESTAMP>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>19</RPC_CALLID>
+      <TIMESTAMP>1512607197680</TIMESTAMP>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>49</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -239,7 +241,7 @@
       <LENGTH>0</LENGTH>
       <INODEID>16388</INODEID>
       <PATH>/directory_mkdir</PATH>
-      <TIMESTAMP>1511309632488</TIMESTAMP>
+      <TIMESTAMP>1512607197690</TIMESTAMP>
       <PERMISSION_STATUS>
         <USERNAME>lei</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
@@ -274,8 +276,8 @@
       <TXID>22</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>24</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>54</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -285,8 +287,8 @@
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
       <SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>25</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>55</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -295,8 +297,8 @@
       <TXID>24</TXID>
       <SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
       <SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>26</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>56</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -306,11 +308,11 @@
       <LENGTH>0</LENGTH>
       <INODEID>16389</INODEID>
       <PATH>/file_create</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632528</MTIME>
-      <ATIME>1511309632528</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197723</MTIME>
+      <ATIME>1512607197723</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -318,8 +320,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>27</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>57</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -329,9 +332,9 @@
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632530</MTIME>
-      <ATIME>1511309632528</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197726</MTIME>
+      <ATIME>1512607197723</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -402,10 +405,10 @@
       <LENGTH>0</LENGTH>
       <SRC>/file_create</SRC>
       <DST>/file_moved</DST>
-      <TIMESTAMP>1511309632561</TIMESTAMP>
+      <TIMESTAMP>1512607197754</TIMESTAMP>
       <OPTIONS>TO_TRASH</OPTIONS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>35</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>65</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -416,10 +419,10 @@
       <INODEID>16390</INODEID>
       <PATH>/file_concat_target</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632566</MTIME>
-      <ATIME>1511309632566</ATIME>
+      <MTIME>1512607197759</MTIME>
+      <ATIME>1512607197759</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -427,8 +430,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>37</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>67</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -533,8 +537,8 @@
       <INODEID>0</INODEID>
       <PATH>/file_concat_target</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632593</MTIME>
-      <ATIME>1511309632566</ATIME>
+      <MTIME>1512607197800</MTIME>
+      <ATIME>1512607197759</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -569,10 +573,10 @@
       <INODEID>16391</INODEID>
       <PATH>/file_concat_0</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632596</MTIME>
-      <ATIME>1511309632596</ATIME>
+      <MTIME>1512607197803</MTIME>
+      <ATIME>1512607197803</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -580,8 +584,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>46</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>76</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -686,8 +691,8 @@
       <INODEID>0</INODEID>
       <PATH>/file_concat_0</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632618</MTIME>
-      <ATIME>1511309632596</ATIME>
+      <MTIME>1512607197837</MTIME>
+      <ATIME>1512607197803</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -722,10 +727,10 @@
       <INODEID>16392</INODEID>
       <PATH>/file_concat_1</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632620</MTIME>
-      <ATIME>1511309632620</ATIME>
+      <MTIME>1512607197839</MTIME>
+      <ATIME>1512607197839</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -733,8 +738,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>55</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>85</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -839,8 +845,8 @@
       <INODEID>0</INODEID>
       <PATH>/file_concat_1</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632643</MTIME>
-      <ATIME>1511309632620</ATIME>
+      <MTIME>1512607197878</MTIME>
+      <ATIME>1512607197839</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -873,13 +879,13 @@
       <TXID>67</TXID>
       <LENGTH>0</LENGTH>
       <TRG>/file_concat_target</TRG>
-      <TIMESTAMP>1511309632648</TIMESTAMP>
+      <TIMESTAMP>1512607197882</TIMESTAMP>
       <SOURCES>
         <SOURCE1>/file_concat_0</SOURCE1>
         <SOURCE2>/file_concat_1</SOURCE2>
       </SOURCES>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>63</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>93</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -890,10 +896,10 @@
       <INODEID>16393</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632651</MTIME>
-      <ATIME>1511309632651</ATIME>
+      <MTIME>1512607197885</MTIME>
+      <ATIME>1512607197885</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -901,8 +907,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>65</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>95</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -974,8 +981,8 @@
       <INODEID>0</INODEID>
       <PATH>/file_create</PATH>
       <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632667</MTIME>
-      <ATIME>1511309632651</ATIME>
+      <MTIME>1512607197909</MTIME>
+      <ATIME>1512607197885</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -1002,10 +1009,10 @@
     <DATA>
       <TXID>76</TXID>
       <SRC>/file_create</SRC>
-      <CLIENTNAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENTNAME>
+      <CLIENTNAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENTNAME>
       <CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
       <NEWLENGTH>512</NEWLENGTH>
-      <TIMESTAMP>1511309632671</TIMESTAMP>
+      <TIMESTAMP>1512607197912</TIMESTAMP>
     </DATA>
   </RECORD>
   <RECORD>
@@ -1016,15 +1023,15 @@
       <INODEID>16394</INODEID>
       <PATH>/file_symlink</PATH>
       <VALUE>/file_concat_target</VALUE>
-      <MTIME>1511309632686</MTIME>
-      <ATIME>1511309632686</ATIME>
+      <MTIME>1512607197921</MTIME>
+      <ATIME>1512607197921</ATIME>
       <PERMISSION_STATUS>
         <USERNAME>lei</USERNAME>
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>511</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>72</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>102</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -1034,11 +1041,11 @@
       <LENGTH>0</LENGTH>
       <INODEID>16395</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309632689</MTIME>
-      <ATIME>1511309632689</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607197925</MTIME>
+      <ATIME>1512607197925</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
-      <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
       <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
       <OVERWRITE>true</OVERWRITE>
       <PERMISSION_STATUS>
@@ -1046,8 +1053,9 @@
         <GROUPNAME>supergroup</GROUPNAME>
         <MODE>420</MODE>
       </PERMISSION_STATUS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>73</RPC_CALLID>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>103</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
@@ -1103,21 +1111,30 @@
     <OPCODE>OP_REASSIGN_LEASE</OPCODE>
     <DATA>
       <TXID>84</TXID>
-      <LEASEHOLDER>DFSClient_NONMAPREDUCE_2134933941_1</LEASEHOLDER>
+      <LEASEHOLDER>DFSClient_NONMAPREDUCE_-923924783_1</LEASEHOLDER>
       <PATH>/hard-lease-recovery-test</PATH>
-      <NEWHOLDER>HDFS_NameNode-2017-11-21 16:13:54,700-0800</NEWHOLDER>
+      <NEWHOLDER>HDFS_NameNode-2017-12-06 16:39:59,951-0800</NEWHOLDER>
     </DATA>
   </RECORD>
   <RECORD>
-    <OPCODE>OP_CLOSE</OPCODE>
+    <OPCODE>OP_REASSIGN_LEASE</OPCODE>
     <DATA>
       <TXID>85</TXID>
+      <LEASEHOLDER>HDFS_NameNode-2017-12-06 16:39:59,951-0800</LEASEHOLDER>
+      <PATH>/hard-lease-recovery-test</PATH>
+      <NEWHOLDER>HDFS_NameNode-2017-12-06 16:40:01,959-0800</NEWHOLDER>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_CLOSE</OPCODE>
+    <DATA>
+      <TXID>86</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
-      <REPLICATION>1</REPLICATION>
-      <MTIME>1511309634880</MTIME>
-      <ATIME>1511309632689</ATIME>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607202974</MTIME>
+      <ATIME>1512607197925</ATIME>
       <BLOCKSIZE>512</BLOCKSIZE>
       <CLIENT_NAME/>
       <CLIENT_MACHINE/>
@@ -1137,7 +1154,7 @@
   <RECORD>
     <OPCODE>OP_ADD_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>86</TXID>
+      <TXID>87</TXID>
       <POOLNAME>pool1</POOLNAME>
       <OWNERNAME>lei</OWNERNAME>
       <GROUPNAME>staff</GROUPNAME>
@@ -1145,65 +1162,65 @@
       <LIMIT>9223372036854775807</LIMIT>
       <MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
       <DEFAULTREPLICATION>1</DEFAULTREPLICATION>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>80</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>138</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>87</TXID>
+      <TXID>88</TXID>
       <POOLNAME>pool1</POOLNAME>
       <LIMIT>99</LIMIT>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>81</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>139</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>88</TXID>
+      <TXID>89</TXID>
       <ID>1</ID>
       <PATH>/path</PATH>
       <REPLICATION>1</REPLICATION>
       <POOL>pool1</POOL>
-      <EXPIRATION>2305844520523329692</EXPIRATION>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>82</RPC_CALLID>
+      <EXPIRATION>2305844521820897941</EXPIRATION>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>140</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>89</TXID>
+      <TXID>90</TXID>
       <ID>1</ID>
       <REPLICATION>2</REPLICATION>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>83</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>141</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
     <DATA>
-      <TXID>90</TXID>
+      <TXID>91</TXID>
       <ID>1</ID>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>84</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>142</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>91</TXID>
+      <TXID>92</TXID>
       <POOLNAME>pool1</POOLNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>85</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>143</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_ACL</OPCODE>
     <DATA>
-      <TXID>92</TXID>
+      <TXID>93</TXID>
       <SRC>/file_concat_target</SRC>
       <ENTRY>
         <SCOPE>ACCESS</SCOPE>
@@ -1236,61 +1253,61 @@
   <RECORD>
     <OPCODE>OP_SET_XATTR</OPCODE>
     <DATA>
-      <TXID>93</TXID>
+      <TXID>94</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a1</NAME>
         <VALUE>0x313233</VALUE>
       </XATTR>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>87</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>145</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_XATTR</OPCODE>
     <DATA>
-      <TXID>94</TXID>
+      <TXID>95</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a2</NAME>
         <VALUE>0x373839</VALUE>
       </XATTR>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>88</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>146</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_XATTR</OPCODE>
     <DATA>
-      <TXID>95</TXID>
+      <TXID>96</TXID>
       <SRC>/file_concat_target</SRC>
       <XATTR>
         <NAMESPACE>USER</NAMESPACE>
         <NAME>a2</NAME>
       </XATTR>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>89</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>147</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>96</TXID>
+      <TXID>97</TXID>
       <CODEC>rs</CODEC>
       <DATAUNITS>3</DATAUNITS>
       <PARITYUNITS>2</PARITYUNITS>
       <CELLSIZE>8192</CELLSIZE>
       <EXTRAOPTIONS>0</EXTRAOPTIONS>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>90</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>148</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ADD_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>97</TXID>
+      <TXID>98</TXID>
       <CODEC>rs</CODEC>
       <DATAUNITS>6</DATAUNITS>
       <PARITYUNITS>10</PARITYUNITS>
@@ -1300,82 +1317,285 @@
         <KEY>dummyKey</KEY>
         <VALUE>dummyValue</VALUE>
       </EXTRAOPTION>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>91</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>149</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>98</TXID>
+      <TXID>99</TXID>
       <POLICYNAME>RS-3-2-8k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>92</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>150</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>99</TXID>
+      <TXID>100</TXID>
       <POLICYNAME>RS-6-10-4k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>93</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>151</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_DISABLE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>100</TXID>
+      <TXID>101</TXID>
       <POLICYNAME>RS-3-2-8k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>94</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>152</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_DISABLE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>101</TXID>
+      <TXID>102</TXID>
       <POLICYNAME>RS-6-10-4k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>95</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>153</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>102</TXID>
+      <TXID>103</TXID>
       <POLICYNAME>RS-3-2-8k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>96</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>154</RPC_CALLID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REMOVE_ERASURE_CODING_POLICY</OPCODE>
     <DATA>
-      <TXID>103</TXID>
+      <TXID>104</TXID>
       <POLICYNAME>RS-6-10-4k</POLICYNAME>
-      <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
-      <RPC_CALLID>97</RPC_CALLID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>155</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_MKDIR</OPCODE>
+    <DATA>
+      <TXID>105</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>16396</INODEID>
+      <PATH>/ec</PATH>
+      <TIMESTAMP>1512607204077</TIMESTAMP>
+      <PERMISSION_STATUS>
+        <USERNAME>lei</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>493</MODE>
+      </PERMISSION_STATUS>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
+    <DATA>
+      <TXID>106</TXID>
+      <POLICYNAME>RS-3-2-1024k</POLICYNAME>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>157</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
+    <DATA>
+      <TXID>107</TXID>
+      <POLICYNAME>RS-6-3-1024k</POLICYNAME>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>158</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_XATTR</OPCODE>
+    <DATA>
+      <TXID>108</TXID>
+      <SRC>/ec</SRC>
+      <XATTR>
+        <NAMESPACE>SYSTEM</NAMESPACE>
+        <NAME>hdfs.erasurecoding.policy</NAME>
+        <VALUE>0x0000000c52532d362d332d313032346b</VALUE>
+      </XATTR>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>159</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD</OPCODE>
+    <DATA>
+      <TXID>109</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>16397</INODEID>
+      <PATH>/ec/replicated</PATH>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607204088</MTIME>
+      <ATIME>1512607204088</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
+      <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+      <OVERWRITE>true</OVERWRITE>
+      <PERMISSION_STATUS>
+        <USERNAME>lei</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+      <ERASURE_CODING_POLICY_ID>0</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>160</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+    <DATA>
+      <TXID>110</TXID>
+      <BLOCK_ID>1073741838</BLOCK_ID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+    <DATA>
+      <TXID>111</TXID>
+      <GENSTAMPV2>1015</GENSTAMPV2>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD_BLOCK</OPCODE>
+    <DATA>
+      <TXID>112</TXID>
+      <PATH>/ec/replicated</PATH>
+      <BLOCK>
+        <BLOCK_ID>1073741838</BLOCK_ID>
+        <NUM_BYTES>0</NUM_BYTES>
+        <GENSTAMP>1015</GENSTAMP>
+      </BLOCK>
+      <RPC_CLIENTID/>
+      <RPC_CALLID>-2</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_CLOSE</OPCODE>
+    <DATA>
+      <TXID>113</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>0</INODEID>
+      <PATH>/ec/replicated</PATH>
+      <REPLICATION>3</REPLICATION>
+      <MTIME>1512607204118</MTIME>
+      <ATIME>1512607204088</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME/>
+      <CLIENT_MACHINE/>
+      <OVERWRITE>false</OVERWRITE>
+      <BLOCK>
+        <BLOCK_ID>1073741838</BLOCK_ID>
+        <NUM_BYTES>10</NUM_BYTES>
+        <GENSTAMP>1015</GENSTAMP>
+      </BLOCK>
+      <PERMISSION_STATUS>
+        <USERNAME>lei</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD</OPCODE>
+    <DATA>
+      <TXID>114</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>16398</INODEID>
+      <PATH>/ec/RS-3-2</PATH>
+      <REPLICATION>1</REPLICATION>
+      <MTIME>1512607204120</MTIME>
+      <ATIME>1512607204120</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME>DFSClient_NONMAPREDUCE_-923924783_1</CLIENT_NAME>
+      <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+      <OVERWRITE>true</OVERWRITE>
+      <PERMISSION_STATUS>
+        <USERNAME>lei</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
+      <ERASURE_CODING_POLICY_ID>2</ERASURE_CODING_POLICY_ID>
+      <RPC_CLIENTID>cab1aa2d-e08a-4d2f-8216-76e167eccd94</RPC_CLIENTID>
+      <RPC_CALLID>166</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+    <DATA>
+      <TXID>115</TXID>
+      <BLOCK_ID>-9223372036854775792</BLOCK_ID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+    <DATA>
+      <TXID>116</TXID>
+      <GENSTAMPV2>1016</GENSTAMPV2>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_ADD_BLOCK</OPCODE>
+    <DATA>
+      <TXID>117</TXID>
+      <PATH>/ec/RS-3-2</PATH>
+      <BLOCK>
+        <BLOCK_ID>-9223372036854775792</BLOCK_ID>
+        <NUM_BYTES>0</NUM_BYTES>
+        <GENSTAMP>1016</GENSTAMP>
+      </BLOCK>
+      <RPC_CLIENTID/>
+      <RPC_CALLID>-2</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_CLOSE</OPCODE>
+    <DATA>
+      <TXID>118</TXID>
+      <LENGTH>0</LENGTH>
+      <INODEID>0</INODEID>
+      <PATH>/ec/RS-3-2</PATH>
+      <REPLICATION>1</REPLICATION>
+      <MTIME>1512607204229</MTIME>
+      <ATIME>1512607204120</ATIME>
+      <BLOCKSIZE>512</BLOCKSIZE>
+      <CLIENT_NAME/>
+      <CLIENT_MACHINE/>
+      <OVERWRITE>false</OVERWRITE>
+      <BLOCK>
+        <BLOCK_ID>-9223372036854775792</BLOCK_ID>
+        <NUM_BYTES>6</NUM_BYTES>
+        <GENSTAMP>1016</GENSTAMP>
+      </BLOCK>
+      <PERMISSION_STATUS>
+        <USERNAME>lei</USERNAME>
+        <GROUPNAME>supergroup</GROUPNAME>
+        <MODE>420</MODE>
+      </PERMISSION_STATUS>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
     <DATA>
-      <TXID>104</TXID>
-      <STARTTIME>1511309635904</STARTTIME>
+      <TXID>119</TXID>
+      <STARTTIME>1512607204230</STARTTIME>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
     <DATA>
-      <TXID>105</TXID>
-      <FINALIZETIME>1511309635904</FINALIZETIME>
+      <TXID>120</TXID>
+      <FINALIZETIME>1512607204233</FINALIZETIME>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_END_LOG_SEGMENT</OPCODE>
     <DATA>
-      <TXID>106</TXID>
+      <TXID>121</TXID>
     </DATA>
   </RECORD>
 </EDITS>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[44/50] hadoop git commit: HDFS-12882. Support full open(PathHandle) contract in HDFS

Posted by xy...@apache.org.
HDFS-12882. Support full open(PathHandle) contract in HDFS


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/693169ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/693169ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/693169ef

Branch: refs/heads/HDFS-7240
Commit: 693169ef34f856a27dc09d90a45fb4ec5b66ed2c
Parents: 55fc2d6
Author: Chris Douglas <cd...@apache.org>
Authored: Mon Dec 11 20:14:15 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Mon Dec 11 20:14:15 2017 -0800

----------------------------------------------------------------------
 .../fs/contract/AbstractContractOpenTest.java   | 53 ++++++++++----
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 74 +++++++++++++++++---
 .../hadoop/hdfs/DistributedFileSystem.java      | 28 +++++---
 .../hadoop/hdfs/protocol/ClientProtocol.java    | 15 ++++
 .../hadoop/hdfs/protocol/HdfsPathHandle.java    | 65 ++++++++++++-----
 .../ClientNamenodeProtocolTranslatorPB.java     | 25 ++++++-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  | 15 ----
 .../src/main/proto/ClientNamenodeProtocol.proto | 11 +++
 .../src/main/proto/hdfs.proto                   |  2 +
 ...tNamenodeProtocolServerSideTranslatorPB.java | 22 ++++++
 .../federation/router/RouterRpcServer.java      | 13 ++++
 .../hdfs/server/namenode/FSDirAppendOp.java     |  3 +-
 .../hdfs/server/namenode/FSDirSnapshotOp.java   |  5 +-
 .../server/namenode/FSDirStatAndListingOp.java  | 41 ++++++-----
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  2 +-
 .../hdfs/server/namenode/FSNamesystem.java      | 16 +++--
 .../hdfs/server/namenode/NameNodeRpcServer.java | 24 +++++--
 .../fsdataset/impl/LazyPersistTestCase.java     |  1 -
 .../hdfs/server/namenode/NameNodeAdapter.java   |  7 +-
 .../hdfs/server/namenode/TestBackupNode.java    |  8 ++-
 .../hdfs/server/namenode/TestEditLog.java       |  3 +-
 .../hdfs/server/namenode/TestReencryption.java  |  5 +-
 .../hdfs/server/namenode/TestSaveNamespace.java |  2 +-
 .../server/namenode/ha/TestEditLogTailer.java   |  4 +-
 .../namenode/ha/TestEditLogsDuringFailover.java | 15 ++--
 .../namenode/ha/TestFailureToReadEdits.java     | 12 ++--
 .../namenode/ha/TestInitializeSharedEdits.java  |  2 +-
 .../namenode/ha/TestStandbyInProgressTail.java  | 69 +++++++++---------
 .../src/test/resources/contract/hdfs.xml        |  2 +-
 29 files changed, 387 insertions(+), 157 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
index 9b6bbd4..7528312 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java
@@ -218,14 +218,18 @@ public abstract class AbstractContractOpenTest
     Path path2 = path("testopenfilebyexact2");
     byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
     createFile(getFileSystem(), path1, false, file1);
-    FileStatus stat = getFileSystem().getFileStatus(path1);
-    assertNotNull(stat);
-    assertEquals(path1, stat.getPath());
+    FileStatus stat1 = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat1);
+    assertEquals(path1, stat1.getPath());
     ContractTestUtils.rename(getFileSystem(), path1, path2);
+    FileStatus stat2 = getFileSystem().getFileStatus(path2);
+    assertNotNull(stat2);
+    assertEquals(path2, stat2.getPath());
     // create identical file at same location, orig still exists at path2
     createFile(getFileSystem(), path1, false, file1);
 
-    PathHandle fd = getHandleOrSkip(stat, HandleOpt.exact());
+    PathHandle fd1 = getHandleOrSkip(stat1, HandleOpt.exact());
+    PathHandle fd2 = getHandleOrSkip(stat2, HandleOpt.exact());
 
     // verify path1, path2 contents identical
     verifyFileContents(getFileSystem(), path1, file1);
@@ -235,11 +239,15 @@ public abstract class AbstractContractOpenTest
       // the original entity exists, it has not been modified, and an
       // identical file exists at the old path. The handle would also
       // fail to resolve if path1 had been modified
-      instream = getFileSystem().open(fd, 1 << 15);
+      instream = getFileSystem().open(fd1);
       fail("Expected an exception");
     } catch (IOException e) {
       // expected
     }
+
+    // verify unchanged resolves
+    instream = getFileSystem().open(fd2);
+    verifyRead(instream, file1, 0, TEST_FILE_LEN);
   }
 
   /**
@@ -265,7 +273,7 @@ public abstract class AbstractContractOpenTest
     // obtain handle to entity from #getFileStatus call
     PathHandle fd = getHandleOrSkip(stat, HandleOpt.content());
 
-    try (FSDataInputStream in = getFileSystem().open(fd, 1 << 15)) {
+    try (FSDataInputStream in = getFileSystem().open(fd)) {
       // verify read of consistent content at new location
       verifyRead(in, file1, 0, TEST_FILE_LEN);
     }
@@ -280,7 +288,7 @@ public abstract class AbstractContractOpenTest
 
     try {
       // handle should not resolve when content changed
-      instream = getFileSystem().open(fd, 1 << 15);
+      instream = getFileSystem().open(fd);
       fail("Failed to detect change to content");
     } catch (IOException e) {
       // expected
@@ -302,25 +310,40 @@ public abstract class AbstractContractOpenTest
 
     byte[] file1 = dataset(TEST_FILE_LEN, 43, 255);
     createFile(getFileSystem(), path1, false, file1);
-    FileStatus stat = getFileSystem().getFileStatus(path1);
-    assertNotNull(stat);
-    assertEquals(path1, stat.getPath());
+    FileStatus stat1 = getFileSystem().getFileStatus(path1);
+    assertNotNull(stat1);
+    assertEquals(path1, stat1.getPath());
     ContractTestUtils.rename(getFileSystem(), path1, path2);
+    FileStatus stat2 = getFileSystem().getFileStatus(path2);
+    assertNotNull(stat2);
+    assertEquals(path2, stat2.getPath());
     // create identical file at same location, orig still exists at path2
     createFile(getFileSystem(), path1, false, file1);
 
-    PathHandle fd = getHandleOrSkip(stat, HandleOpt.path());
+    PathHandle fd1 = getHandleOrSkip(stat1, HandleOpt.path());
+    PathHandle fd2 = getHandleOrSkip(stat2, HandleOpt.path());
 
     // verify path1, path2 contents identical
     verifyFileContents(getFileSystem(), path1, file1);
     verifyFileContents(getFileSystem(), path2, file1);
     try {
       // verify attempt to resolve the handle fails
-      instream = getFileSystem().open(fd, 1 << 15);
+      instream = getFileSystem().open(fd1);
       fail("Expected an exception");
     } catch (IOException e) {
       // expected
     }
+
+    // verify content change OK
+    byte[] file2a = dataset(TEST_FILE_LEN, 44, 255);
+    ContractTestUtils.appendFile(getFileSystem(), path2, file2a);
+    byte[] file2x = Arrays.copyOf(file1, file1.length + file2a.length);
+    System.arraycopy(file2a, 0, file2x, file1.length, file2a.length);
+    // verify path2 contains contents of orig + appended bytes
+    verifyFileContents(getFileSystem(), path2, file2x);
+    // verify open by fd succeeds
+    instream = getFileSystem().open(fd2);
+    verifyRead(instream, file2x, 0, 2 * TEST_FILE_LEN);
   }
 
   /**
@@ -357,8 +380,8 @@ public abstract class AbstractContractOpenTest
     verifyFileContents(getFileSystem(), path1, file2);
 
     // verify fd contains contents of file1 + appended bytes
-    instream = getFileSystem().open(fd, 1 << 15);
-    verifyRead(instream, file1x, 0, TEST_FILE_LEN);
+    instream = getFileSystem().open(fd);
+    verifyRead(instream, file1x, 0, 2 * TEST_FILE_LEN);
   }
 
   /**
@@ -388,7 +411,7 @@ public abstract class AbstractContractOpenTest
     ByteBuffer sb = fd.bytes();
     PathHandle fdb = new RawPathHandle(sb);
 
-    instream = getFileSystem().open(fdb, 1 << 15);
+    instream = getFileSystem().open(fdb);
     // verify stat contains contents of file1
     verifyRead(instream, file1, 0, TEST_FILE_LEN);
     // verify path2 contains contents of file1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 51dff8d..59f553b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -125,6 +125,8 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -1015,16 +1017,46 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     //    Get block info from namenode
     try (TraceScope ignored = newPathTraceScope("newDFSInputStream", src)) {
       LocatedBlocks locatedBlocks = getLocatedBlocks(src, 0);
-      if (locatedBlocks != null) {
-        ErasureCodingPolicy ecPolicy = locatedBlocks.getErasureCodingPolicy();
-        if (ecPolicy != null) {
-          return new DFSStripedInputStream(this, src, verifyChecksum, ecPolicy,
-              locatedBlocks);
-        }
-        return new DFSInputStream(this, src, verifyChecksum, locatedBlocks);
-      } else {
-        throw new IOException("Cannot open filename " + src);
+      return openInternal(locatedBlocks, src, verifyChecksum);
+    }
+  }
+
+  /**
+   * Create an input stream from the {@link HdfsPathHandle} if the
+   * constraints encoded from {@link
+   * DistributedFileSystem#createPathHandle(FileStatus, Options.HandleOpt...)}
+   * are satisfied. Note that HDFS does not ensure that these constraints
+   * remain invariant for the life of the stream. It only checks that they
+   * still held when the stream was opened.
+   * @param fd Handle to an entity in HDFS, with constraints
+   * @param buffersize ignored
+   * @param verifyChecksum Verify checksums before returning data to client
+   * @return Data from the referent of the {@link HdfsPathHandle}.
+   * @throws IOException On I/O error
+   */
+  public DFSInputStream open(HdfsPathHandle fd, int buffersize,
+      boolean verifyChecksum) throws IOException {
+    checkOpen();
+    String src = fd.getPath();
+    try (TraceScope ignored = newPathTraceScope("newDFSInputStream", src)) {
+      HdfsLocatedFileStatus s = getLocatedFileInfo(src, true);
+      fd.verify(s); // check invariants in path handle
+      LocatedBlocks locatedBlocks = s.getLocatedBlocks();
+      return openInternal(locatedBlocks, src, verifyChecksum);
+    }
+  }
+
+  private DFSInputStream openInternal(LocatedBlocks locatedBlocks, String src,
+      boolean verifyChecksum) throws IOException {
+    if (locatedBlocks != null) {
+      ErasureCodingPolicy ecPolicy = locatedBlocks.getErasureCodingPolicy();
+      if (ecPolicy != null) {
+        return new DFSStripedInputStream(this, src, verifyChecksum, ecPolicy,
+            locatedBlocks);
       }
+      return new DFSInputStream(this, src, verifyChecksum, locatedBlocks);
+    } else {
+      throw new IOException("Cannot open filename " + src);
     }
   }
 
@@ -1648,6 +1680,30 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
   }
 
   /**
+   * Get the file info for a specific file or directory.
+   * @param src The string representation of the path to the file
+   * @param needBlockToken Include block tokens in {@link LocatedBlocks}.
+   *        When block tokens are included, this call is a superset of
+   *        {@link #getBlockLocations(String, long)}.
+   * @return object containing information regarding the file
+   *         or null if file not found
+   *
+   * @see DFSClient#open(HdfsPathHandle, int, boolean)
+   * @see ClientProtocol#getFileInfo(String) for description of
+   *      exceptions
+   */
+  public HdfsLocatedFileStatus getLocatedFileInfo(String src,
+      boolean needBlockToken) throws IOException {
+    checkOpen();
+    try (TraceScope ignored = newPathTraceScope("getLocatedFileInfo", src)) {
+      return namenode.getLocatedFileInfo(src, needBlockToken);
+    } catch (RemoteException re) {
+      throw re.unwrapRemoteException(AccessControlException.class,
+          FileNotFoundException.class,
+          UnresolvedPathException.class);
+    }
+  }
+  /**
    * Close status of a file
    * @return true if file is already closed
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 1478868..2449fb6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -115,7 +115,7 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.Optional;
 
 /****************************************************************
  * Implementation of the abstract FileSystem for the DFS system.
@@ -340,11 +340,14 @@ public class DistributedFileSystem extends FileSystem
   @Override
   public FSDataInputStream open(PathHandle fd, int bufferSize)
       throws IOException {
+    statistics.incrementReadOps(1);
+    storageStatistics.incrementOpCounter(OpType.OPEN);
     if (!(fd instanceof HdfsPathHandle)) {
       fd = new HdfsPathHandle(fd.bytes());
     }
     HdfsPathHandle id = (HdfsPathHandle) fd;
-    return open(DFSUtilClient.makePathFromFileId(id.getInodeId()), bufferSize);
+    final DFSInputStream dfsis = dfs.open(id, bufferSize, verifyChecksum);
+    return dfs.createWrappedInputStream(dfsis);
   }
 
   /**
@@ -358,7 +361,7 @@ public class DistributedFileSystem extends FileSystem
    * @return A handle to the file.
    */
   @Override
-  protected PathHandle createPathHandle(FileStatus st, HandleOpt... opts) {
+  protected HdfsPathHandle createPathHandle(FileStatus st, HandleOpt... opts) {
     if (!(st instanceof HdfsFileStatus)) {
       throw new IllegalArgumentException("Invalid FileStatus "
           + st.getClass().getSimpleName());
@@ -373,12 +376,21 @@ public class DistributedFileSystem extends FileSystem
         .orElse(HandleOpt.changed(false));
     HandleOpt.Location loc = HandleOpt.getOpt(HandleOpt.Location.class, opts)
         .orElse(HandleOpt.moved(false));
-    if (!data.allowChange() || !loc.allowChange()) {
-      throw new UnsupportedOperationException("Unsupported opts "
-          + Arrays.stream(opts)
-                  .map(HandleOpt::toString).collect(Collectors.joining(",")));
+
+    HdfsFileStatus hst = (HdfsFileStatus) st;
+    final Path p;
+    final Optional<Long> inodeId;
+    if (loc.allowChange()) {
+      p = DFSUtilClient.makePathFromFileId(hst.getFileId());
+      inodeId = Optional.empty();
+    } else {
+      p = hst.getPath();
+      inodeId = Optional.of(hst.getFileId());
     }
-    return new HdfsPathHandle((HdfsFileStatus)st);
+    final Optional<Long> mtime = !data.allowChange()
+        ? Optional.of(hst.getModificationTime())
+        : Optional.empty();
+    return new HdfsPathHandle(getPathName(p), inodeId, mtime);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index eb2e11c..e8a33dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1025,6 +1025,21 @@ public interface ClientProtocol {
   HdfsFileStatus getFileLinkInfo(String src) throws IOException;
 
   /**
+   * Get the file info for a specific file or directory with
+   * {@link LocatedBlocks}.
+   * @param src The string representation of the path to the file
+   * @param needBlockToken Generate block tokens for {@link LocatedBlocks}
+   * @return object containing information regarding the file
+   *         or null if file not found
+   * @throws org.apache.hadoop.security.AccessControlException permission denied
+   * @throws java.io.FileNotFoundException file <code>src</code> is not found
+   * @throws IOException If an I/O error occurred
+   */
+  @Idempotent
+  HdfsLocatedFileStatus getLocatedFileInfo(String src, boolean needBlockToken)
+      throws IOException;
+
+  /**
    * Get {@link ContentSummary} rooted at the specified directory.
    * @param path The string representation of the path
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
index 2d8dcca..f80a067 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsPathHandle.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocol;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Optional;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -34,16 +35,17 @@ import com.google.protobuf.ByteString;
 @InterfaceStability.Unstable
 public final class HdfsPathHandle implements PathHandle {
 
-  private static final long serialVersionUID = 0xc5308795428L;
+  private static final long serialVersionUID = 0xc53087a5428L;
 
-  private final long inodeId;
+  private final String path;
+  private final Long mtime;
+  private final Long inodeId;
 
-  public HdfsPathHandle(HdfsFileStatus hstat) {
-    this(hstat.getFileId());
-  }
-
-  public HdfsPathHandle(long inodeId) {
-    this.inodeId = inodeId;
+  public HdfsPathHandle(String path,
+      Optional<Long> inodeId, Optional<Long> mtime) {
+    this.path = path;
+    this.mtime = mtime.orElse(null);
+    this.inodeId = inodeId.orElse(null);
   }
 
   public HdfsPathHandle(ByteBuffer bytes) throws IOException {
@@ -52,20 +54,39 @@ public final class HdfsPathHandle implements PathHandle {
     }
     HdfsPathHandleProto p =
         HdfsPathHandleProto.parseFrom(ByteString.copyFrom(bytes));
-    inodeId = p.getInodeId();
+    path = p.getPath();
+    mtime = p.hasMtime()
+        ? p.getMtime()
+        : null;
+    inodeId = p.hasInodeId()
+        ? p.getInodeId()
+        : null;
   }
 
-  public long getInodeId() {
-    return inodeId;
+  public String getPath() {
+    return path;
+  }
+
+  public void verify(HdfsLocatedFileStatus stat) throws IOException {
+    if (mtime != null && mtime != stat.getModificationTime()) {
+      throw new IOException("Content changed");
+    }
+    if (inodeId != null && inodeId != stat.getFileId()) {
+      throw new IOException("Wrong file");
+    }
   }
 
   @Override
   public ByteBuffer bytes() {
-    return HdfsPathHandleProto.newBuilder()
-      .setInodeId(getInodeId())
-      .build()
-      .toByteString()
-      .asReadOnlyByteBuffer();
+    HdfsPathHandleProto.Builder b = HdfsPathHandleProto.newBuilder();
+    b.setPath(path);
+    if (inodeId != null) {
+      b.setInodeId(inodeId);
+    }
+    if (mtime != null) {
+      b.setMtime(mtime);
+    }
+    return b.build().toByteString().asReadOnlyByteBuffer();
   }
 
   @Override
@@ -78,19 +99,25 @@ public final class HdfsPathHandle implements PathHandle {
       return false;
     }
     HdfsPathHandle o = (HdfsPathHandle)other;
-    return getInodeId() == o.getInodeId();
+    return getPath().equals(o.getPath());
   }
 
   @Override
   public int hashCode() {
-    return Long.hashCode(inodeId);
+    return path.hashCode();
   }
 
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("{ ");
-    sb.append("inodeId : ").append(Long.toString(getInodeId()));
+    sb.append("\"path\" : \"").append(path).append("\"");
+    if (inodeId != null) {
+      sb.append(",\"inodeId\" : ").append(inodeId);
+    }
+    if (mtime != null) {
+      sb.append(",\"mtime\" : ").append(mtime);
+    }
     sb.append(" }");
     return sb.toString();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 38dc44b..9ccc2fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -129,6 +130,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLin
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLocatedFileInfoRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLocatedFileInfoResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetServerDefaultsRequestProto;
@@ -872,7 +875,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
   @Override
   public HdfsFileStatus getFileInfo(String src) throws IOException {
     GetFileInfoRequestProto req = GetFileInfoRequestProto.newBuilder()
-        .setSrc(src).build();
+        .setSrc(src)
+        .build();
     try {
       GetFileInfoResponseProto res = rpcProxy.getFileInfo(null, req);
       return res.hasFs() ? PBHelperClient.convert(res.getFs()) : null;
@@ -882,6 +886,25 @@ public class ClientNamenodeProtocolTranslatorPB implements
   }
 
   @Override
+  public HdfsLocatedFileStatus getLocatedFileInfo(String src,
+      boolean needBlockToken) throws IOException {
+    GetLocatedFileInfoRequestProto req =
+        GetLocatedFileInfoRequestProto.newBuilder()
+            .setSrc(src)
+            .setNeedBlockToken(needBlockToken)
+            .build();
+    try {
+      GetLocatedFileInfoResponseProto res =
+          rpcProxy.getLocatedFileInfo(null, req);
+      return (HdfsLocatedFileStatus) (res.hasFs()
+          ? PBHelperClient.convert(res.getFs())
+          : null);
+    } catch (ServiceException e) {
+      throw ProtobufHelper.getRemoteException(e);
+    }
+  }
+
+  @Override
   public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
     GetFileLinkInfoRequestProto req = GetFileLinkInfoRequestProto.newBuilder()
         .setSrc(src).build();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index fbc6dbf..73c5215 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -91,7 +91,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -164,7 +163,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsServerDefaultsProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsPathHandleProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
@@ -1624,19 +1622,6 @@ public class PBHelperClient {
     return FsPermissionProto.newBuilder().setPerm(p.toShort()).build();
   }
 
-  public static HdfsPathHandle convert(HdfsPathHandleProto fd) {
-    if (null == fd) {
-      return null;
-    }
-    return new HdfsPathHandle(fd.getInodeId());
-  }
-
-  public static HdfsPathHandleProto convert(HdfsPathHandle fd) {
-    return HdfsPathHandleProto.newBuilder()
-        .setInodeId(fd.getInodeId())
-        .build();
-  }
-
   public static HdfsFileStatus convert(HdfsFileStatusProto fs) {
     if (fs == null) {
       return null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index eb6da25..b33462b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -495,6 +495,15 @@ message GetFileInfoResponseProto {
   optional HdfsFileStatusProto fs = 1;
 }
 
+message GetLocatedFileInfoRequestProto {
+  optional string src = 1;
+  optional bool needBlockToken = 2 [default = false];
+}
+
+message GetLocatedFileInfoResponseProto {
+  optional HdfsFileStatusProto fs = 1;
+}
+
 message IsFileClosedRequestProto {
   required string src = 1;
 }
@@ -868,6 +877,8 @@ service ClientNamenodeProtocol {
       returns(ListCorruptFileBlocksResponseProto);
   rpc metaSave(MetaSaveRequestProto) returns(MetaSaveResponseProto);
   rpc getFileInfo(GetFileInfoRequestProto) returns(GetFileInfoResponseProto);
+  rpc getLocatedFileInfo(GetLocatedFileInfoRequestProto)
+      returns(GetLocatedFileInfoResponseProto);
   rpc addCacheDirective(AddCacheDirectiveRequestProto)
       returns (AddCacheDirectiveResponseProto);
   rpc modifyCacheDirective(ModifyCacheDirectiveRequestProto)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index a423a4b..fca1e0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -401,6 +401,8 @@ message AddErasureCodingPolicyResponseProto {
  */
 message HdfsPathHandleProto {
   optional uint64 inodeId = 1;
+  optional uint64 mtime = 2;
+  optional string path = 3;
 }
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 2ae41e4..d63460b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -136,6 +136,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLin
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLocatedFileInfoRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLocatedFileInfoResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetQuotaUsageRequestProto;
@@ -344,6 +346,10 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   private static final GetFileInfoResponseProto VOID_GETFILEINFO_RESPONSE = 
   GetFileInfoResponseProto.newBuilder().build();
 
+  private static final GetLocatedFileInfoResponseProto
+      VOID_GETLOCATEDFILEINFO_RESPONSE =
+          GetLocatedFileInfoResponseProto.newBuilder().build();
+
   private static final GetFileLinkInfoResponseProto VOID_GETFILELINKINFO_RESPONSE = 
   GetFileLinkInfoResponseProto.newBuilder().build();
 
@@ -952,7 +958,23 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
   }
 
   @Override
+  public GetLocatedFileInfoResponseProto getLocatedFileInfo(
+      RpcController controller, GetLocatedFileInfoRequestProto req)
+      throws ServiceException {
+    try {
+      HdfsFileStatus result = server.getLocatedFileInfo(req.getSrc(),
+          req.getNeedBlockToken());
+      if (result != null) {
+        return GetLocatedFileInfoResponseProto.newBuilder().setFs(
+            PBHelperClient.convert(result)).build();
+      }
+      return VOID_GETLOCATEDFILEINFO_RESPONSE;
+    } catch (IOException e) {
+      throw new ServiceException(e);
+    }
+  }
 
+  @Override
   public GetFileLinkInfoResponseProto getFileLinkInfo(RpcController controller,
       GetFileLinkInfoRequestProto req) throws ServiceException {
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index d401e69..c6cd595 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -85,6 +85,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -1071,6 +1072,18 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
         locations, method, HdfsFileStatus.class, null);
   }
 
+  @Override
+  public HdfsLocatedFileStatus getLocatedFileInfo(String src,
+      boolean needBlockToken) throws IOException {
+    checkOperation(OperationCategory.READ);
+    final List<RemoteLocation> locations = getLocationsForPath(src, false);
+    RemoteMethod method = new RemoteMethod("getLocatedFileInfo",
+        new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
+        Boolean.valueOf(needBlockToken));
+    return (HdfsLocatedFileStatus) rpcClient.invokeSequential(
+        locations, method, HdfsFileStatus.class, null);
+  }
+
   @Override // ClientProtocol
   public long[] getStats() throws IOException {
     checkOperation(OperationCategory.UNCHECKED);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 9926ee0..be272d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -148,7 +148,8 @@ final class FSDirAppendOp {
       fsd.writeUnlock();
     }
 
-    HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, iip);
+    HdfsFileStatus stat =
+        FSDirStatAndListingOp.getFileInfo(fsd, iip, false, false);
     if (lb != null) {
       NameNode.stateChangeLog.debug(
           "DIR* NameSystem.appendFile: file {} for {} at {} block {} block"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
index 1842707..4dacbf2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.protocol.FSLimitException;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReportListing;
 import org.apache.hadoop.hdfs.protocol.SnapshotException;
@@ -213,7 +214,9 @@ class FSDirSnapshotOp {
           snapname += Path.SEPARATOR;
         }
         snapname += file.substring(file.indexOf(dirName) + dirName.length());
-        if (fsd.getFSNamesystem().getFileInfo(snapname, true) != null) {
+        HdfsFileStatus stat =
+            fsd.getFSNamesystem().getFileInfo(snapname, true, false, false);
+        if (stat != null) {
           snaps.add(snapname);
         }
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 4da8a28..8b77034 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -90,11 +90,13 @@ class FSDirStatAndListingOp {
    * @param resolveLink whether to throw UnresolvedLinkException
    *        if src refers to a symlink
    *
+   * @param needLocation Include {@link LocatedBlocks} in result.
+   * @param needBlockToken Include block tokens in {@link LocatedBlocks}.
    * @return object containing information regarding the file
    *         or null if file not found
    */
-  static HdfsFileStatus getFileInfo(
-      FSDirectory fsd, String srcArg, boolean resolveLink)
+  static HdfsFileStatus getFileInfo(FSDirectory fsd, String srcArg,
+      boolean resolveLink, boolean needLocation, boolean needBlockToken)
       throws IOException {
     DirOp dirOp = resolveLink ? DirOp.READ : DirOp.READ_LINK;
     FSPermissionChecker pc = fsd.getPermissionChecker();
@@ -111,7 +113,7 @@ class FSDirStatAndListingOp {
     } else {
       iip = fsd.resolvePath(pc, srcArg, dirOp);
     }
-    return getFileInfo(fsd, iip);
+    return getFileInfo(fsd, iip, needLocation, needBlockToken);
   }
 
   /**
@@ -234,7 +236,7 @@ class FSDirStatAndListingOp {
         // target INode
         return new DirectoryListing(
             new HdfsFileStatus[]{ createFileStatus(
-                fsd, iip, null, parentStoragePolicy, needLocation)
+                fsd, iip, null, parentStoragePolicy, needLocation, false)
             }, 0);
       }
 
@@ -253,8 +255,8 @@ class FSDirStatAndListingOp {
             ? getStoragePolicyID(child.getLocalStoragePolicyID(),
                                  parentStoragePolicy)
             : parentStoragePolicy;
-        listing[i] =
-            createFileStatus(fsd, iip, child, childStoragePolicy, needLocation);
+        listing[i] = createFileStatus(fsd, iip, child, childStoragePolicy,
+            needLocation, false);
         listingCnt++;
         if (listing[i] instanceof HdfsLocatedFileStatus) {
             // Once we  hit lsLimit locations, stop.
@@ -305,7 +307,7 @@ class FSDirStatAndListingOp {
     for (int i = 0; i < numOfListing; i++) {
       Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot();
       listing[i] = createFileStatus(fsd, iip, sRoot,
-          HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false);
+          HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false, false);
     }
     return new DirectoryListing(
         listing, snapshots.size() - skipSize - numOfListing);
@@ -324,11 +326,14 @@ class FSDirStatAndListingOp {
    * @param fsd FSDirectory
    * @param iip The path to the file, the file is included
    * @param includeStoragePolicy whether to include storage policy
+   * @param needLocation Include {@link LocatedBlocks} in response
+   * @param needBlockToken Generate block tokens for {@link LocatedBlocks}
    * @return object containing information regarding the file
    *         or null if file not found
    */
-  static HdfsFileStatus getFileInfo(FSDirectory fsd,
-      INodesInPath iip, boolean includeStoragePolicy) throws IOException {
+  static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip,
+      boolean includeStoragePolicy, boolean needLocation,
+      boolean needBlockToken) throws IOException {
     fsd.readLock();
     try {
       final INode node = iip.getLastINode();
@@ -338,14 +343,15 @@ class FSDirStatAndListingOp {
       byte policy = (includeStoragePolicy && !node.isSymlink())
           ? node.getStoragePolicyID()
           : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
-      return createFileStatus(fsd, iip, null, policy, false);
+      return createFileStatus(fsd, iip, null, policy, needLocation,
+          needBlockToken);
     } finally {
       fsd.readUnlock();
     }
   }
 
-  static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip)
-    throws IOException {
+  static HdfsFileStatus getFileInfo(FSDirectory fsd, INodesInPath iip,
+      boolean needLocation, boolean needBlockToken) throws IOException {
     fsd.readLock();
     try {
       HdfsFileStatus status = null;
@@ -356,7 +362,7 @@ class FSDirStatAndListingOp {
           status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
         }
       } else {
-        status = getFileInfo(fsd, iip, true);
+        status = getFileInfo(fsd, iip, true, needLocation, needBlockToken);
       }
       return status;
     } finally {
@@ -373,7 +379,7 @@ class FSDirStatAndListingOp {
   static HdfsFileStatus createFileStatusForEditLog(
       FSDirectory fsd, INodesInPath iip) throws IOException {
     return createFileStatus(fsd, iip,
-        null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false);
+        null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, false, false);
   }
 
   /**
@@ -384,12 +390,13 @@ class FSDirStatAndListingOp {
    * @param child for a directory listing of the iip, else null
    * @param storagePolicy for the path or closest ancestor
    * @param needLocation if block locations need to be included or not
+   * @param needBlockToken
    * @return a file status
    * @throws java.io.IOException if any error occurs
    */
   private static HdfsFileStatus createFileStatus(
       FSDirectory fsd, INodesInPath iip, INode child, byte storagePolicy,
-      boolean needLocation) throws IOException {
+      boolean needLocation, boolean needBlockToken) throws IOException {
     assert fsd.hasReadLock();
     // only directory listing sets the status name.
     byte[] name = HdfsFileStatus.EMPTY_NAME;
@@ -429,8 +436,8 @@ class FSDirStatAndListingOp {
         final long fileSize = !inSnapshot && isUc
             ? fileNode.computeFileSizeNotIncludingLastUcBlock() : size;
         loc = fsd.getBlockManager().createLocatedBlocks(
-            fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false,
-            inSnapshot, feInfo, ecPolicy);
+            fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size,
+            needBlockToken, inSnapshot, feInfo, ecPolicy);
         if (loc == null) {
           loc = new LocatedBlocks();
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index c4041a3..8f34e1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -408,7 +408,7 @@ class FSDirWriteFileOp {
       NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
           src + " inode " + newNode.getId() + " " + holder);
     }
-    return FSDirStatAndListingOp.getFileInfo(fsd, iip);
+    return FSDirStatAndListingOp.getFileInfo(fsd, iip, false, false);
   }
 
   static INodeFile addFileForEditLog(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 6a890e2..dedb737 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -2990,6 +2990,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    * @param resolveLink whether to throw UnresolvedLinkException
    *        if src refers to a symlink
    *
+   * @param needLocation Include {@link LocatedBlocks} in result.
+   * @param needBlockToken Include block tokens in {@link LocatedBlocks}
    * @throws AccessControlException if access is denied
    * @throws UnresolvedLinkException if a symlink is encountered.
    *
@@ -2997,15 +2999,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    *         or null if file not found
    * @throws StandbyException
    */
-  HdfsFileStatus getFileInfo(final String src, boolean resolveLink)
-    throws IOException {
-    final String operationName = "getfileinfo";
+  HdfsFileStatus getFileInfo(final String src, boolean resolveLink,
+      boolean needLocation, boolean needBlockToken) throws IOException {
+    // if the client requests block tokens, then it can read data blocks
+    // and should appear in the audit log as if getBlockLocations had been
+    // called
+    final String operationName = needBlockToken ? "open" : "getfileinfo";
     checkOperation(OperationCategory.READ);
     HdfsFileStatus stat = null;
     readLock();
     try {
       checkOperation(OperationCategory.READ);
-      stat = FSDirStatAndListingOp.getFileInfo(dir, src, resolveLink);
+      stat = FSDirStatAndListingOp.getFileInfo(
+          dir, src, resolveLink, needLocation, needBlockToken);
     } catch (AccessControlException e) {
       logAuditEvent(false, operationName, src);
       throw e;
@@ -6158,7 +6164,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
     }
 
     for (CorruptFileBlockInfo c : corruptFileBlocks) {
-      if (getFileInfo(c.path, true) != null) {
+      if (getFileInfo(c.path, true, false, false) != null) {
         list.add(c.toString());
       }
       final Collection<String> snaps = FSDirSnapshotOp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 36d33a6..94bd15f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.FSLimitException;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
 import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
@@ -1138,12 +1139,25 @@ public class NameNodeRpcServer implements NamenodeProtocols {
   }
 
   @Override // ClientProtocol
-  public HdfsFileStatus getFileInfo(String src)  throws IOException {
+  public HdfsFileStatus getFileInfo(String src) throws IOException {
     checkNNStartup();
     metrics.incrFileInfoOps();
-    return namesystem.getFileInfo(src, true);
+    return namesystem.getFileInfo(src, true, false, false);
   }
-  
+
+  @Override // ClientProtocol
+  public HdfsLocatedFileStatus getLocatedFileInfo(String src,
+      boolean needBlockToken) throws IOException {
+    checkNNStartup();
+    if (needBlockToken) {
+      metrics.incrGetBlockLocations();
+    } else {
+      metrics.incrFileInfoOps();
+    }
+    return (HdfsLocatedFileStatus)
+        namesystem.getFileInfo(src, true, true, needBlockToken);
+  }
+
   @Override // ClientProtocol
   public boolean isFileClosed(String src) throws IOException{
     checkNNStartup();
@@ -1154,7 +1168,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
   public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
     checkNNStartup();
     metrics.incrFileInfoOps();
-    return namesystem.getFileInfo(src, false);
+    return namesystem.getFileInfo(src, false, false, false);
   }
   
   @Override // ClientProtocol
@@ -1429,7 +1443,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
     metrics.incrGetLinkTargetOps();
     HdfsFileStatus stat = null;
     try {
-      stat = namesystem.getFileInfo(path, false);
+      stat = namesystem.getFileInfo(path, false, false, false);
     } catch (UnresolvedPathException e) {
       return e.getResolvedPath().toString();
     } catch (UnresolvedLinkException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index 799d5d1..c412dad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
 import com.google.common.base.Supplier;
-import org.apache.commons.lang.UnhandledException;
 import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 
 import static org.apache.hadoop.fs.CreateFlag.CREATE;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
index 242e8f5..11d7959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
@@ -72,12 +72,13 @@ public class NameNodeAdapter {
   }
   
   public static HdfsFileStatus getFileInfo(NameNode namenode, String src,
-      boolean resolveLink) throws AccessControlException, UnresolvedLinkException,
-        StandbyException, IOException {
+      boolean resolveLink, boolean needLocation, boolean needBlockToken)
+      throws AccessControlException, UnresolvedLinkException, StandbyException,
+      IOException {
     namenode.getNamesystem().readLock();
     try {
       return FSDirStatAndListingOp.getFileInfo(namenode.getNamesystem()
-          .getFSDirectory(), src, resolveLink);
+          .getFSDirectory(), src, resolveLink, needLocation, needBlockToken);
     } finally {
       namenode.getNamesystem().readUnlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
index 10d9f11..2678964 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBackupNode.java
@@ -274,7 +274,8 @@ public class TestBackupNode {
       backup = startBackupNode(conf, StartupOption.BACKUP, 1);
 
       testBNInSync(cluster, backup, 4);
-      assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down", false));
+      assertNotNull(backup.getNamesystem()
+          .getFileInfo("/edit-while-bn-down", false, false, false));
       
       // Trigger an unclean shutdown of the backup node. Backup node will not
       // unregister from the active when this is done simulating a node crash.
@@ -314,7 +315,8 @@ public class TestBackupNode {
         public Boolean get() {
           LOG.info("Checking for " + src + " on BN");
           try {
-            boolean hasFile = backup.getNamesystem().getFileInfo(src, false) != null;
+            boolean hasFile = backup.getNamesystem()
+                .getFileInfo(src, false, false, false) != null;
             boolean txnIdMatch =
               backup.getRpcServer().getTransactionID() ==
               nn.getRpcServer().getTransactionID();
@@ -465,7 +467,7 @@ public class TestBackupNode {
       assertTrue("file3 does not exist on BackupNode",
           op != StartupOption.BACKUP ||
           backup.getNamesystem().getFileInfo(
-              file3.toUri().getPath(), false) != null);
+              file3.toUri().getPath(), false, false, false) != null);
 
     } catch(IOException e) {
       LOG.error("Error in TestBackupNode:", e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 8bc8a77..ce52221 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -288,7 +288,8 @@ public class TestEditLog {
       long numEdits = testLoad(HADOOP20_SOME_EDITS, namesystem);
       assertEquals(3, numEdits);
       // Sanity check the edit
-      HdfsFileStatus fileInfo = namesystem.getFileInfo("/myfile", false);
+      HdfsFileStatus fileInfo =
+          namesystem.getFileInfo("/myfile", false, false, false);
       assertEquals("supergroup", fileInfo.getGroup());
       assertEquals(3, fileInfo.getReplication());
     } finally {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
index aca9a73..f0d6834 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -163,7 +163,8 @@ public class TestReencryption {
   }
 
   private FileEncryptionInfo getFileEncryptionInfo(Path path) throws Exception {
-    return fsn.getFileInfo(path.toString(), false).getFileEncryptionInfo();
+    return fsn.getFileInfo(path.toString(), false, false, false)
+        .getFileEncryptionInfo();
   }
 
   @Test
@@ -1954,4 +1955,4 @@ public class TestReencryption {
     // after NN  restart consistent.
     dfsAdmin.getKeyProvider().flush();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index 0c86ef4..5602954 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -745,7 +745,7 @@ public class TestSaveNamespace {
 
   private void checkEditExists(FSNamesystem fsn, int id) throws IOException {
     // Make sure the image loaded including our edit.
-    assertNotNull(fsn.getFileInfo("/test" + id, false));
+    assertNotNull(fsn.getFileInfo("/test" + id, false, false, false));
   }
 
   private Configuration getConf() throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index 9d218c2..61f890c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -124,7 +124,7 @@ public class TestEditLogTailer {
       
       for (int i = 0; i < DIRS_TO_MAKE / 2; i++) {
         assertTrue(NameNodeAdapter.getFileInfo(nn2,
-            getDirPath(i), false).isDirectory());
+            getDirPath(i), false, false, false).isDirectory());
       }
       
       for (int i = DIRS_TO_MAKE / 2; i < DIRS_TO_MAKE; i++) {
@@ -137,7 +137,7 @@ public class TestEditLogTailer {
       
       for (int i = DIRS_TO_MAKE / 2; i < DIRS_TO_MAKE; i++) {
         assertTrue(NameNodeAdapter.getFileInfo(nn2,
-            getDirPath(i), false).isDirectory());
+            getDirPath(i), false, false, false).isDirectory());
       }
     } finally {
       cluster.shutdown();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
index a8d350d..4405c5b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java
@@ -38,9 +38,10 @@ import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.test.GenericTestUtils;
+import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getFileInfo;
+
 import org.junit.Test;
 
 import com.google.common.base.Joiner;
@@ -110,7 +111,8 @@ public class TestEditLogsDuringFailover {
       // the current log segment, and on the next roll, it would have to
       // either replay starting in the middle of the segment (not allowed)
       // or double-replay the edits (incorrect).
-      assertNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1), "/test", true));
+      assertNull(getFileInfo(cluster.getNameNode(1), "/test",
+          true, false, false));
       
       cluster.getNameNode(0).getRpcServer().mkdirs("/test2",
           FsPermission.createImmutable((short)0755), true);
@@ -122,8 +124,10 @@ public class TestEditLogsDuringFailover {
 
       // NN1 should have both the edits that came before its restart, and the edits that
       // came after its restart.
-      assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1), "/test", true));
-      assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1), "/test2", true));
+      assertNotNull(getFileInfo(cluster.getNameNode(1), "/test",
+          true, false, false));
+      assertNotNull(getFileInfo(cluster.getNameNode(1), "/test2",
+          true, false, false));
     } finally {
       cluster.shutdown();
     }
@@ -165,7 +169,8 @@ public class TestEditLogsDuringFailover {
       // In the transition to active, it should have read the log -- and
       // hence see one of the dirs we made in the fake log.
       String testPath = "/dir" + NUM_DIRS_IN_LOG;
-      assertNotNull(cluster.getNameNode(0).getRpcServer().getFileInfo(testPath));
+      assertNotNull(cluster.getNameNode(0).getRpcServer()
+          .getFileInfo(testPath));
       
       // It also should have finalized that log in the shared directory and started
       // writing to a new one at the next txid.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
index a37631f..284e283 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
@@ -213,13 +213,13 @@ public class TestFailureToReadEdits {
     
     // Null because it was deleted.
     assertNull(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR1, false));
+        TEST_DIR1, false, false, false));
     // Should have been successfully created.
     assertTrue(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR2, false).isDirectory());
+        TEST_DIR2, false, false, false).isDirectory());
     // Null because it hasn't been created yet.
     assertNull(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR3, false));
+        TEST_DIR3, false, false, false));
 
     // Now let the standby read ALL the edits.
     answer.setThrowExceptionOnRead(false);
@@ -227,13 +227,13 @@ public class TestFailureToReadEdits {
     
     // Null because it was deleted.
     assertNull(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR1, false));
+        TEST_DIR1, false, false, false));
     // Should have been successfully created.
     assertTrue(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR2, false).isDirectory());
+        TEST_DIR2, false, false, false).isDirectory());
     // Should now have been successfully created.
     assertTrue(NameNodeAdapter.getFileInfo(nn1,
-        TEST_DIR3, false).isDirectory());
+        TEST_DIR3, false, false, false).isDirectory());
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
index 8eeb853..5015f9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestInitializeSharedEdits.java
@@ -128,7 +128,7 @@ public class TestInitializeSharedEdits {
       HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
           cluster.getNameNode(1));
       assertTrue(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-          newPath.toString(), false).isDirectory());
+          newPath.toString(), false, false, false).isDirectory());
     } finally {
       if (fs != null) {
         fs.close();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java
index b1cd037..5da8178 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyInProgressTail.java
@@ -35,8 +35,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
 import org.apache.hadoop.test.GenericTestUtils;
+import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getFileInfo;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -116,8 +117,8 @@ public class TestStandbyInProgressTail {
       cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
 
       // StandbyNameNode should not finish tailing in-progress logs
-      assertNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-              "/test", true));
+      assertNull(getFileInfo(cluster.getNameNode(1),
+              "/test", true, false, false));
 
       // Restarting the standby should not finalize any edits files
       // in the shared directory when it starts up!
@@ -132,8 +133,8 @@ public class TestStandbyInProgressTail {
       // the current log segment, and on the next roll, it would have to
       // either replay starting in the middle of the segment (not allowed)
       // or double-replay the edits (incorrect).
-      assertNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-              "/test", true));
+      assertNull(getFileInfo(cluster.getNameNode(1),
+              "/test", true, false, false));
 
       cluster.getNameNode(0).getRpcServer().mkdirs("/test2",
               FsPermission.createImmutable((short) 0755), true);
@@ -145,10 +146,10 @@ public class TestStandbyInProgressTail {
 
       // NN1 should have both the edits that came before its restart,
       // and the edits that came after its restart.
-      assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-              "/test", true));
-      assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-              "/test2", true));
+      assertNotNull(getFileInfo(cluster.getNameNode(1),
+              "/test", true, false, false));
+      assertNotNull(getFileInfo(cluster.getNameNode(1),
+              "/test2", true, false, false));
     } finally {
       if (qjmhaCluster != null) {
         qjmhaCluster.shutdown();
@@ -182,8 +183,8 @@ public class TestStandbyInProgressTail {
 
     // After waiting for 5 seconds, StandbyNameNode should finish tailing
     // in-progress logs
-    assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-            "/test", true));
+    assertNotNull(getFileInfo(cluster.getNameNode(1),
+            "/test", true, false, false));
 
     // Restarting the standby should not finalize any edits files
     // in the shared directory when it starts up!
@@ -194,8 +195,8 @@ public class TestStandbyInProgressTail {
     assertNoEditFiles(cluster.getNameDirs(1));
 
     // Because we're using in-progress tailer, this should not be null
-    assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-            "/test", true));
+    assertNotNull(getFileInfo(cluster.getNameNode(1),
+            "/test", true, false, false));
 
     cluster.getNameNode(0).getRpcServer().mkdirs("/test2",
             FsPermission.createImmutable((short) 0755), true);
@@ -207,10 +208,10 @@ public class TestStandbyInProgressTail {
 
     // NN1 should have both the edits that came before its restart,
     // and the edits that came after its restart.
-    assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-            "/test", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(cluster.getNameNode(1),
-            "/test2", true));
+    assertNotNull(getFileInfo(cluster.getNameNode(1),
+            "/test", true, false, false));
+    assertNotNull(getFileInfo(cluster.getNameNode(1),
+            "/test2", true, false, false));
   }
 
   @Test
@@ -229,7 +230,7 @@ public class TestStandbyInProgressTail {
     nn1.getNamesystem().getEditLogTailer().doTailEdits();
 
     // StandbyNameNode should tail the in-progress edit
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
 
     // Create a new edit and finalized it
     cluster.getNameNode(0).getRpcServer().mkdirs("/test2",
@@ -237,7 +238,7 @@ public class TestStandbyInProgressTail {
     nn0.getRpcServer().rollEditLog();
 
     // StandbyNameNode shouldn't tail the edit since we do not call the method
-    assertNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
+    assertNull(getFileInfo(nn1, "/test2", true, false, false));
 
     // Create a new in-progress edit and let SBNN do the tail
     cluster.getNameNode(0).getRpcServer().mkdirs("/test3",
@@ -245,9 +246,9 @@ public class TestStandbyInProgressTail {
     nn1.getNamesystem().getEditLogTailer().doTailEdits();
 
     // StandbyNameNode should tail the finalized edit and the new in-progress
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test3", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test2", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test3", true, false, false));
   }
 
   @Test
@@ -270,16 +271,16 @@ public class TestStandbyInProgressTail {
     cluster.getNameNode(0).getRpcServer().mkdirs("/test3",
             FsPermission.createImmutable((short) 0755), true);
 
-    assertNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
-    assertNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
-    assertNull(NameNodeAdapter.getFileInfo(nn1, "/test3", true));
+    assertNull(getFileInfo(nn1, "/test", true, false, false));
+    assertNull(getFileInfo(nn1, "/test2", true, false, false));
+    assertNull(getFileInfo(nn1, "/test3", true, false, false));
 
     nn1.getNamesystem().getEditLogTailer().doTailEdits();
 
     // StandbyNameNode shoudl tail the finalized edit and the new in-progress
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test3", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test2", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test3", true, false, false));
   }
 
   @Test
@@ -296,17 +297,17 @@ public class TestStandbyInProgressTail {
             FsPermission.createImmutable((short) 0755), true);
     nn1.getNamesystem().getEditLogTailer().doTailEdits();
     nn0.getRpcServer().rollEditLog();
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test2", true, false, false));
 
     cluster.getNameNode(0).getRpcServer().mkdirs("/test3",
             FsPermission.createImmutable((short) 0755), true);
     nn1.getNamesystem().getEditLogTailer().doTailEdits();
 
     // StandbyNameNode shoudl tail the finalized edit and the new in-progress
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test2", true));
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test3", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test2", true, false, false));
+    assertNotNull(getFileInfo(nn1, "/test3", true, false, false));
   }
 
   @Test
@@ -325,7 +326,7 @@ public class TestStandbyInProgressTail {
     cluster.getNameNode(0).getRpcServer().rollEdits();
 
     cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
-    assertNotNull(NameNodeAdapter.getFileInfo(nn1, "/test", true));
+    assertNotNull(getFileInfo(nn1, "/test", true, false, false));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/693169ef/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
index d38d3d8..261d4ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml
@@ -108,7 +108,7 @@
 
   <property>
     <name>fs.contract.supports-content-check</name>
-    <value>false</value>
+    <value>true</value>
   </property>
 
 </configuration>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[41/50] hadoop git commit: MAPREDUCE-7018. Apply erasure coding properly to framework tarball and support plain tar (miklos.szegedi@cloudera.com via rkanter)

Posted by xy...@apache.org.
MAPREDUCE-7018. Apply erasure coding properly to framework tarball and support plain tar (miklos.szegedi@cloudera.com via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2316f526
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2316f526
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2316f526

Branch: refs/heads/HDFS-7240
Commit: 2316f526902e827b6c1b92a5bddef72d211bc742
Parents: 00129c5
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Dec 11 14:00:42 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon Dec 11 14:00:42 2017 -0800

----------------------------------------------------------------------
 .../mapred/uploader/FrameworkUploader.java      | 59 +++++++++------
 .../mapred/uploader/TestFrameworkUploader.java  | 79 +++++++++++---------
 2 files changed, 79 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2316f526/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index d1cd740..a374262 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -81,7 +81,6 @@ public class FrameworkUploader implements Runnable {
 
   @VisibleForTesting
   OutputStream targetStream = null;
-  private Path targetPath = null;
   private String alias = null;
 
   private void printHelp(Options options) {
@@ -140,11 +139,12 @@ public class FrameworkUploader implements Runnable {
     }
   }
 
-  private void beginUpload() throws IOException, UploaderException {
+  @VisibleForTesting
+  void beginUpload() throws IOException, UploaderException {
     if (targetStream == null) {
       validateTargetPath();
       int lastIndex = target.indexOf('#');
-      targetPath =
+      Path targetPath =
           new Path(
               target.substring(
                   0, lastIndex == -1 ? target.length() : lastIndex));
@@ -153,7 +153,37 @@ public class FrameworkUploader implements Runnable {
           targetPath.getName();
       LOG.info("Target " + targetPath);
       FileSystem fileSystem = targetPath.getFileSystem(new Configuration());
-      targetStream = fileSystem.create(targetPath, true);
+
+      targetStream = null;
+      if (fileSystem instanceof DistributedFileSystem) {
+        LOG.info("Set replication to " +
+            replication + " for path: " + targetPath);
+        LOG.info("Disabling Erasure Coding for path: " + targetPath);
+        DistributedFileSystem dfs = (DistributedFileSystem)fileSystem;
+        DistributedFileSystem.HdfsDataOutputStreamBuilder builder =
+            dfs.createFile(targetPath)
+            .overwrite(true)
+            .ecPolicyName(
+                SystemErasureCodingPolicies.getReplicationPolicy().getName());
+        if (replication > 0) {
+          builder.replication(replication);
+        }
+        targetStream = builder.build();
+      } else {
+        LOG.warn("Cannot set replication to " +
+            replication + " for path: " + targetPath +
+            " on a non-distributed fileystem " +
+            fileSystem.getClass().getName());
+      }
+      if (targetStream == null) {
+        targetStream = fileSystem.create(targetPath, true);
+      }
+
+      if (targetPath.getName().endsWith("gz") ||
+          targetPath.getName().endsWith("tgz")) {
+        LOG.info("Creating GZip");
+        targetStream = new GZIPOutputStream(targetStream);
+      }
     }
   }
 
@@ -162,7 +192,7 @@ public class FrameworkUploader implements Runnable {
     beginUpload();
     LOG.info("Compressing tarball");
     try (TarArchiveOutputStream out = new TarArchiveOutputStream(
-        new GZIPOutputStream(targetStream))) {
+        targetStream)) {
       for (String fullPath : filteredInputFiles) {
         LOG.info("Adding " + fullPath);
         File file = new File(fullPath);
@@ -178,25 +208,6 @@ public class FrameworkUploader implements Runnable {
         targetStream.close();
       }
     }
-
-    if (targetPath == null) {
-      return;
-    }
-
-    // Set file attributes
-    FileSystem fileSystem = targetPath.getFileSystem(new Configuration());
-    if (fileSystem instanceof DistributedFileSystem) {
-      LOG.info("Disabling Erasure Coding for path: " + targetPath);
-      DistributedFileSystem dfs = (DistributedFileSystem) fileSystem;
-      dfs.setErasureCodingPolicy(targetPath,
-          SystemErasureCodingPolicies.getReplicationPolicy().getName());
-    }
-
-    if (replication > 0) {
-      LOG.info("Set replication to " +
-          replication + " for path: " + targetPath);
-      fileSystem.setReplication(targetPath, replication);
-    }
   }
 
   private void parseLists() throws UploaderException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2316f526/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
index 9d03165..f3e4fc5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
@@ -30,6 +30,7 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -171,46 +172,54 @@ public class TestFrameworkUploader {
    */
   @Test
   public void testBuildTarBall() throws IOException, UploaderException {
-    File parent = new File(testDir);
-    try {
-      parent.deleteOnExit();
-      FrameworkUploader uploader = prepareTree(parent);
+    String[] testFiles = {"upload.tar", "upload.tar.gz"};
+    for (String testFile: testFiles) {
+      File parent = new File(testDir);
+      try {
+        parent.deleteOnExit();
+        FrameworkUploader uploader = prepareTree(parent);
 
-      File gzipFile = new File("upload.tar.gz");
-      gzipFile.deleteOnExit();
-      Assert.assertTrue("Creating output", gzipFile.createNewFile());
-      uploader.targetStream = new FileOutputStream(gzipFile);
+        File gzipFile =
+            new File(parent.getAbsolutePath() + "/" + testFile);
+        gzipFile.deleteOnExit();
 
-      uploader.buildPackage();
+        uploader.target =
+            "file:///" + gzipFile.getAbsolutePath();
+        uploader.beginUpload();
+        uploader.buildPackage();
+        InputStream stream = new FileInputStream(gzipFile);
+        if (gzipFile.getName().endsWith(".gz")) {
+          stream = new GZIPInputStream(stream);
+        }
 
-      TarArchiveInputStream result = null;
-      try {
-        result =
-            new TarArchiveInputStream(
-                new GZIPInputStream(new FileInputStream(gzipFile)));
-        Set<String> fileNames = new HashSet<>();
-        Set<Long> sizes = new HashSet<>();
-        TarArchiveEntry entry1 = result.getNextTarEntry();
-        fileNames.add(entry1.getName());
-        sizes.add(entry1.getSize());
-        TarArchiveEntry entry2 = result.getNextTarEntry();
-        fileNames.add(entry2.getName());
-        sizes.add(entry2.getSize());
-        Assert.assertTrue(
-            "File name error", fileNames.contains("a.jar"));
-        Assert.assertTrue(
-            "File size error", sizes.contains((long) 13));
-        Assert.assertTrue(
-            "File name error", fileNames.contains("b.jar"));
-        Assert.assertTrue(
-            "File size error", sizes.contains((long) 14));
-      } finally {
-        if (result != null) {
-          result.close();
+        TarArchiveInputStream result = null;
+        try {
+          result =
+              new TarArchiveInputStream(stream);
+          Set<String> fileNames = new HashSet<>();
+          Set<Long> sizes = new HashSet<>();
+          TarArchiveEntry entry1 = result.getNextTarEntry();
+          fileNames.add(entry1.getName());
+          sizes.add(entry1.getSize());
+          TarArchiveEntry entry2 = result.getNextTarEntry();
+          fileNames.add(entry2.getName());
+          sizes.add(entry2.getSize());
+          Assert.assertTrue(
+              "File name error", fileNames.contains("a.jar"));
+          Assert.assertTrue(
+              "File size error", sizes.contains((long) 13));
+          Assert.assertTrue(
+              "File name error", fileNames.contains("b.jar"));
+          Assert.assertTrue(
+              "File size error", sizes.contains((long) 14));
+        } finally {
+          if (result != null) {
+            result.close();
+          }
         }
+      } finally {
+        FileUtils.deleteDirectory(parent);
       }
-    } finally {
-      FileUtils.deleteDirectory(parent);
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[45/50] hadoop git commit: YARN-7635. TestRMWebServicesSchedulerActivities fails in trunk. Contributed by Sunil G.

Posted by xy...@apache.org.
YARN-7635. TestRMWebServicesSchedulerActivities fails in trunk. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c87fb2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c87fb2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c87fb2f

Branch: refs/heads/HDFS-7240
Commit: 5c87fb2f627ee64f5713d25cbea4a8956ed4de9c
Parents: 693169e
Author: Sunil G <su...@apache.org>
Authored: Tue Dec 12 15:07:54 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Dec 12 15:08:18 2017 +0530

----------------------------------------------------------------------
 .../resourcemanager/webapp/TestRMWebServicesCapacitySched.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c87fb2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 0132348..edf0652 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -357,7 +357,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     int numExpectedElements = 20;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 33;
+      numExpectedElements = 34;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[27/50] hadoop git commit: HDFS-12825. Fsck report shows config key name for min replication issues (Contributed by Gabor Bota).

Posted by xy...@apache.org.
HDFS-12825. Fsck report shows config key name for min replication issues (Contributed by Gabor Bota).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef7d334d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef7d334d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef7d334d

Branch: refs/heads/HDFS-7240
Commit: ef7d334d364378070880e647eaf8bac2f12561ee
Parents: f196383
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Fri Dec 8 11:46:30 2017 -0800
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Fri Dec 8 11:46:30 2017 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java  | 2 +-
 .../test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7d334d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index b6d6971..a82c9fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -1236,7 +1236,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
                 ((float) (numUnderMinReplicatedBlocks * 100) / (float) totalBlocks))
                 .append(" %)");
           }
-          res.append("\n  ").append(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY + ":\t")
+          res.append("\n  ").append("MINIMAL BLOCK REPLICATION:\t")
              .append(minReplication);
         }
         if(corruptFiles>0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef7d334d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 0c04738..f80fd70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -902,7 +902,7 @@ public class TestFsck {
     System.out.println(outStr);
     assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
     assertTrue(outStr.contains("UNDER MIN REPL'D BLOCKS:\t1 (100.0 %)"));
-    assertTrue(outStr.contains("dfs.namenode.replication.min:\t2"));
+    assertTrue(outStr.contains("MINIMAL BLOCK REPLICATION:\t2"));
   }
 
   @Test(timeout = 90000)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[09/50] hadoop git commit: HADOOP-15098. TestClusterTopology#testChooseRandom fails intermittently. Contributed by Zsolt Venczel.

Posted by xy...@apache.org.
HADOOP-15098. TestClusterTopology#testChooseRandom fails intermittently. Contributed by Zsolt Venczel.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acb92904
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acb92904
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acb92904

Branch: refs/heads/HDFS-7240
Commit: acb92904d04053551d3010937e133f74450043df
Parents: 67662e2
Author: Sean Mackrory <ma...@apache.org>
Authored: Thu Dec 7 10:52:02 2017 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Dec 7 12:30:58 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/net/TestClusterTopology.java  | 58 ++++++++++++--------
 1 file changed, 35 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/acb92904/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
index 09f0914..6aad6c5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
@@ -139,38 +139,50 @@ public class TestClusterTopology extends Assert {
     NodeElement node4 = getNewNode("node4", "/d1/r3");
     cluster.add(node4);
 
+    // Number of test runs
+    int numTestRuns = 3;
+    int chiSquareTestRejectedCounter = 0;
+
     // Number of iterations to do the test
     int numIterations = 100;
 
-    // Pick random nodes
-    HashMap<String,Integer> histogram = new HashMap<String,Integer>();
-    for (int i=0; i<numIterations; i++) {
-      String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName();
-      if (!histogram.containsKey(randomNode)) {
-        histogram.put(randomNode, 0);
+    for (int testRun = 0; testRun < numTestRuns; ++testRun) {
+
+      // Pick random nodes
+      HashMap<String, Integer> histogram = new HashMap<String, Integer>();
+      for (int i = 0; i < numIterations; i++) {
+        String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName();
+        if (!histogram.containsKey(randomNode)) {
+          histogram.put(randomNode, 0);
+        }
+        histogram.put(randomNode, histogram.get(randomNode) + 1);
+      }
+      assertEquals("Random is not selecting all nodes", 4, histogram.size());
+
+      // Check with 99% confidence alpha=0.01 as confidence = 100 * (1 - alpha)
+      ChiSquareTest chiSquareTest = new ChiSquareTest();
+      double[] expected = new double[histogram.size()];
+      long[] observed = new long[histogram.size()];
+      int j = 0;
+      for (Integer occurrence : histogram.values()) {
+        expected[j] = 1.0 * numIterations / histogram.size();
+        observed[j] = occurrence;
+        j++;
+      }
+      boolean chiSquareTestRejected =
+            chiSquareTest.chiSquareTest(expected, observed, 0.01);
+
+      if (chiSquareTestRejected) {
+        ++chiSquareTestRejectedCounter;
       }
-      histogram.put(randomNode, histogram.get(randomNode) + 1);
-    }
-    assertEquals("Random is not selecting all nodes", 4, histogram.size());
-
-    // Check with 99% confidence (alpha=0.01 as confidence = (100 * (1 - alpha)
-    ChiSquareTest chiSquareTest = new ChiSquareTest();
-    double[] expected = new double[histogram.size()];
-    long[] observed = new long[histogram.size()];
-    int j=0;
-    for (Integer occurrence : histogram.values()) {
-      expected[j] = 1.0 * numIterations / histogram.size();
-      observed[j] = occurrence;
-      j++;
     }
-    boolean chiSquareTestRejected =
-        chiSquareTest.chiSquareTest(expected, observed, 0.01);
 
     // Check that they have the proper distribution
-    assertFalse("Not choosing nodes randomly", chiSquareTestRejected);
+    assertFalse("Random not choosing nodes with proper distribution",
+            chiSquareTestRejectedCounter==3);
 
     // Pick random nodes excluding the 2 nodes in /d1/r3
-    histogram = new HashMap<String,Integer>();
+    HashMap<String, Integer> histogram = new HashMap<String, Integer>();
     for (int i=0; i<numIterations; i++) {
       String randomNode = cluster.chooseRandom("~/d1/r3").getName();
       if (!histogram.containsKey(randomNode)) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[14/50] hadoop git commit: YARN-7482. Max applications calculation per queue has to be retrospected with absolute resource support. Contributed by Sunil G.

Posted by xy...@apache.org.
YARN-7482. Max applications calculation per queue has to be retrospected with absolute resource support. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e65ca92f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e65ca92f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e65ca92f

Branch: refs/heads/HDFS-7240
Commit: e65ca92fb6897a3004a235d7b69e308399189aae
Parents: 034b312
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Nov 16 17:33:00 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/ParentQueue.java         | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e65ca92f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 940637e..a427fb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1088,6 +1088,26 @@ public class ParentQueue extends AbstractCSQueue {
     childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
         (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
             / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+
+    // Re-visit max applications for a queue based on absolute capacity if
+    // needed.
+    if (childQueue instanceof LeafQueue) {
+      LeafQueue leafQueue = (LeafQueue) childQueue;
+      CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+      int maxApplications = (int) (conf.getMaximumSystemApplications()
+          * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+      leafQueue.setMaxApplications(maxApplications);
+
+      int maxApplicationsPerUser = Math.min(maxApplications,
+          (int) (maxApplications
+              * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
+              * leafQueue.getUsersManager().getUserLimitFactor()));
+      leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
+      LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications="
+          + maxApplications + ", maxApplicationsPerUser="
+          + maxApplicationsPerUser + ", Abs Cap:"
+          + childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+    }
   }
 
   @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[23/50] hadoop git commit: YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)

Change-Id: I9213f5297a6841fab5c573e85ee4c4e5f4a0b7ff


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e798b1a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e798b1a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e798b1a

Branch: refs/heads/HDFS-7240
Commit: 5e798b1a0ddceeaf54703b94052501867156e979
Parents: d6c31a3
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 11 10:30:23 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/util/StringUtils.java     |  31 ++
 .../resource/DefaultResourceCalculator.java     |   6 +
 .../resource/DominantResourceCalculator.java    |   7 +
 .../yarn/util/resource/ResourceCalculator.java  |  12 +
 .../hadoop/yarn/util/resource/Resources.java    |   5 +
 .../capacity/FifoCandidatesSelector.java        |   9 +-
 .../ProportionalCapacityPreemptionPolicy.java   |  10 +-
 .../monitor/capacity/TempQueuePerPartition.java |  16 +-
 .../scheduler/AbstractResourceUsage.java        | 198 +++++++
 .../scheduler/QueueResourceQuotas.java          | 153 ++++++
 .../scheduler/ResourceUsage.java                | 237 ++-------
 .../scheduler/capacity/AbstractCSQueue.java     | 162 +++++-
 .../scheduler/capacity/CSQueue.java             |  39 ++
 .../scheduler/capacity/CSQueueUtils.java        |  24 +-
 .../CapacitySchedulerConfiguration.java         | 179 ++++++-
 .../scheduler/capacity/LeafQueue.java           |  31 +-
 .../scheduler/capacity/ParentQueue.java         | 201 +++++++-
 .../scheduler/capacity/UsersManager.java        |   5 +-
 .../PriorityUtilizationQueueOrderingPolicy.java |  11 +
 .../webapp/dao/CapacitySchedulerQueueInfo.java  |  15 +
 .../yarn/server/resourcemanager/MockNM.java     |   8 +
 .../yarn/server/resourcemanager/MockRM.java     |   6 +
 ...alCapacityPreemptionPolicyMockFramework.java |  13 +
 ...estProportionalCapacityPreemptionPolicy.java |  28 +-
 ...pacityPreemptionPolicyIntraQueueWithDRF.java |   6 +-
 .../TestAbsoluteResourceConfiguration.java      | 516 +++++++++++++++++++
 .../capacity/TestApplicationLimits.java         |  30 +-
 .../TestApplicationLimitsByPartition.java       |   4 +
 .../capacity/TestCapacityScheduler.java         |   2 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   2 +
 .../scheduler/capacity/TestLeafQueue.java       | 261 ++++------
 .../scheduler/capacity/TestParentQueue.java     |   8 +
 .../scheduler/capacity/TestReservations.java    |  19 +
 ...tPriorityUtilizationQueueOrderingPolicy.java |   3 +
 .../webapp/TestRMWebServicesCapacitySched.java  |   4 +-
 35 files changed, 1831 insertions(+), 430 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index cda5ec7..1be8a08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1152,4 +1152,35 @@ public class StringUtils {
     return s1.equalsIgnoreCase(s2);
   }
 
+  /**
+   * <p>Checks if the String contains only unicode letters.</p>
+   *
+   * <p><code>null</code> will return <code>false</code>.
+   * An empty String (length()=0) will return <code>true</code>.</p>
+   *
+   * <pre>
+   * StringUtils.isAlpha(null)   = false
+   * StringUtils.isAlpha("")     = true
+   * StringUtils.isAlpha("  ")   = false
+   * StringUtils.isAlpha("abc")  = true
+   * StringUtils.isAlpha("ab2c") = false
+   * StringUtils.isAlpha("ab-c") = false
+   * </pre>
+   *
+   * @param str  the String to check, may be null
+   * @return <code>true</code> if only contains letters, and is non-null
+   */
+  public static boolean isAlpha(String str) {
+      if (str == null) {
+          return false;
+      }
+      int sz = str.length();
+      for (int i = 0; i < sz; i++) {
+          if (Character.isLetter(str.charAt(i)) == false) {
+              return false;
+          }
+      }
+      return true;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 7f155e7..aefa85c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -131,4 +131,10 @@ public class DefaultResourceCalculator extends ResourceCalculator {
   public boolean isAnyMajorResourceZero(Resource resource) {
     return resource.getMemorySize() == 0f;
   }
+
+  @Override
+  public Resource normalizeDown(Resource r, Resource stepFactor) {
+    return Resources.createResource(
+        roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6b284e3..16e4527 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -567,4 +567,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
     }
     return false;
   }
+
+  @Override
+  public Resource normalizeDown(Resource r, Resource stepFactor) {
+    return Resources.createResource(
+        roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
+        roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index d59560f..a816290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -235,4 +235,16 @@ public abstract class ResourceCalculator {
    * @return returns true if any resource is zero.
    */
   public abstract boolean isAnyMajorResourceZero(Resource resource);
+
+  /**
+   * Get resource <code>r</code>and normalize down using step-factor
+   * <code>stepFactor</code>.
+   *
+   * @param r
+   *          resource to be multiplied
+   * @param stepFactor
+   *          factor by which to normalize down
+   * @return resulting normalized resource
+   */
+  public abstract Resource normalizeDown(Resource r, Resource stepFactor);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 068e7f1..0426ab7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -546,4 +546,9 @@ public class Resources {
       Resource resource) {
     return rc.isAnyMajorResourceZero(resource);
   }
+
+  public static Resource normalizeDown(ResourceCalculator calculator,
+      Resource resource, Resource factor) {
+    return calculator.normalizeDown(resource, factor);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index f843db4..748548a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -140,10 +140,10 @@ public class FifoCandidatesSelector
         // Can try preempting AMContainers (still saving atmost
         // maxAMCapacityForThisQueue AMResource's) if more resources are
         // required to be preemptionCandidates from this Queue.
-        Resource maxAMCapacityForThisQueue = Resources.multiply(
-            Resources.multiply(clusterResource,
-                leafQueue.getAbsoluteCapacity()),
-            leafQueue.getMaxAMResourcePerQueuePercent());
+        Resource maxAMCapacityForThisQueue = Resources
+            .multiply(
+                leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
+                leafQueue.getMaxAMResourcePerQueuePercent());
 
         preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
             resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
@@ -199,7 +199,6 @@ public class FifoCandidatesSelector
    * Given a target preemption for a specific application, select containers
    * to preempt (after unreserving all reservation for that app).
    */
-  @SuppressWarnings("unchecked")
   private void preemptFrom(FiCaSchedulerApp app,
       Resource clusterResource, Map<String, Resource> resToObtainByPartition,
       List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 2c072d2..8327cb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolic
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -525,6 +526,13 @@ public class ProportionalCapacityPreemptionPolicy
       float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt);
       boolean preemptionDisabled = curQueue.getPreemptionDisabled();
 
+      QueueResourceQuotas queueResourceQuotas = curQueue
+          .getQueueResourceQuotas();
+      Resource effMinRes = queueResourceQuotas
+          .getEffectiveMinResource(partitionToLookAt);
+      Resource effMaxRes = queueResourceQuotas
+          .getEffectiveMaxResource(partitionToLookAt);
+
       Resource current = Resources
           .clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt));
       Resource killable = Resources.none();
@@ -550,7 +558,7 @@ public class ProportionalCapacityPreemptionPolicy
 
       ret = new TempQueuePerPartition(queueName, current, preemptionDisabled,
           partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
-          reserved, curQueue);
+          reserved, curQueue, effMinRes, effMaxRes);
 
       if (curQueue instanceof ParentQueue) {
         String configuredOrderingPolicy =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 89452f9..bd236fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -48,6 +48,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
 
   double normalizedGuarantee;
 
+  private Resource effMinRes;
+  private Resource effMaxRes;
+
   final ArrayList<TempQueuePerPartition> children;
   private Collection<TempAppPerPartition> apps;
   LeafQueue leafQueue;
@@ -68,7 +71,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
   TempQueuePerPartition(String queueName, Resource current,
       boolean preemptionDisabled, String partition, Resource killable,
       float absCapacity, float absMaxCapacity, Resource totalPartitionResource,
-      Resource reserved, CSQueue queue) {
+      Resource reserved, CSQueue queue, Resource effMinRes,
+      Resource effMaxRes) {
     super(queueName, current, Resource.newInstance(0, 0), reserved,
         Resource.newInstance(0, 0));
 
@@ -95,6 +99,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
     this.absCapacity = absCapacity;
     this.absMaxCapacity = absMaxCapacity;
     this.totalPartitionResource = totalPartitionResource;
+    this.effMinRes = effMinRes;
+    this.effMaxRes = effMaxRes;
   }
 
   public void setLeafQueue(LeafQueue l) {
@@ -177,10 +183,18 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
   }
 
   public Resource getGuaranteed() {
+    if(!effMinRes.equals(Resources.none())) {
+      return Resources.clone(effMinRes);
+    }
+
     return Resources.multiply(totalPartitionResource, absCapacity);
   }
 
   public Resource getMax() {
+    if(!effMaxRes.equals(Resources.none())) {
+      return Resources.clone(effMaxRes);
+    }
+
     return Resources.multiply(totalPartitionResource, absMaxCapacity);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
new file mode 100644
index 0000000..c295323
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class AbstractResourceUsage {
+  protected ReadLock readLock;
+  protected WriteLock writeLock;
+  protected Map<String, UsageByLabel> usages;
+  // short for no-label :)
+  private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+  public AbstractResourceUsage() {
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    readLock = lock.readLock();
+    writeLock = lock.writeLock();
+
+    usages = new HashMap<String, UsageByLabel>();
+    usages.put(NL, new UsageByLabel(NL));
+  }
+
+  // Usage enum here to make implement cleaner
+  public enum ResourceType {
+    // CACHED_USED and CACHED_PENDING may be read by anyone, but must only
+    // be written by ordering policies
+    USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
+        5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
+            9), EFF_MAX_RESOURCE(
+                10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+
+    private int idx;
+
+    private ResourceType(int value) {
+      this.idx = value;
+    }
+  }
+
+  public static class UsageByLabel {
+    // usage by label, contains all UsageType
+    private Resource[] resArr;
+
+    public UsageByLabel(String label) {
+      resArr = new Resource[ResourceType.values().length];
+      for (int i = 0; i < resArr.length; i++) {
+        resArr[i] = Resource.newInstance(0, 0);
+      };
+    }
+
+    public Resource getUsed() {
+      return resArr[ResourceType.USED.idx];
+    }
+
+    @Override
+    public String toString() {
+      StringBuilder sb = new StringBuilder();
+      sb.append("{used=" + resArr[0] + "%, ");
+      sb.append("pending=" + resArr[1] + "%, ");
+      sb.append("am_used=" + resArr[2] + "%, ");
+      sb.append("reserved=" + resArr[3] + "%}");
+      sb.append("min_eff=" + resArr[9] + "%, ");
+      sb.append("max_eff=" + resArr[10] + "%}");
+      sb.append("min_effup=" + resArr[11] + "%, ");
+      return sb.toString();
+    }
+  }
+
+  private static Resource normalize(Resource res) {
+    if (res == null) {
+      return Resources.none();
+    }
+    return res;
+  }
+
+  protected Resource _get(String label, ResourceType type) {
+    if (label == null) {
+      label = RMNodeLabelsManager.NO_LABEL;
+    }
+
+    try {
+      readLock.lock();
+      UsageByLabel usage = usages.get(label);
+      if (null == usage) {
+        return Resources.none();
+      }
+      return normalize(usage.resArr[type.idx]);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  protected Resource _getAll(ResourceType type) {
+    try {
+      readLock.lock();
+      Resource allOfType = Resources.createResource(0);
+      for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
+        //all usages types are initialized
+        Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+      }
+      return allOfType;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private UsageByLabel getAndAddIfMissing(String label) {
+    if (label == null) {
+      label = RMNodeLabelsManager.NO_LABEL;
+    }
+    if (!usages.containsKey(label)) {
+      UsageByLabel u = new UsageByLabel(label);
+      usages.put(label, u);
+      return u;
+    }
+
+    return usages.get(label);
+  }
+
+  protected void _set(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      usage.resArr[type.idx] = res;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  protected void _inc(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      Resources.addTo(usage.resArr[type.idx], res);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  protected void _dec(String label, ResourceType type, Resource res) {
+    try {
+      writeLock.lock();
+      UsageByLabel usage = getAndAddIfMissing(label);
+      Resources.subtractFrom(usage.resArr[type.idx], res);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  @Override
+  public String toString() {
+    try {
+      readLock.lock();
+      return usages.toString();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public Set<String> getNodePartitionsSet() {
+    try {
+      readLock.lock();
+      return usages.keySet();
+    } finally {
+      readLock.unlock();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
new file mode 100644
index 0000000..2e653fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+/**
+ * QueueResourceQuotas by Labels for following fields by label
+ * - EFFECTIVE_MIN_CAPACITY
+ * - EFFECTIVE_MAX_CAPACITY
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class QueueResourceQuotas extends AbstractResourceUsage {
+  // short for no-label :)
+  private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+  public QueueResourceQuotas() {
+    super();
+  }
+
+  /*
+   * Configured Minimum Resource
+   */
+  public Resource getConfiguredMinResource() {
+    return _get(NL, ResourceType.MIN_RESOURCE);
+  }
+
+  public Resource getConfiguredMinResource(String label) {
+    return _get(label, ResourceType.MIN_RESOURCE);
+  }
+
+  public void setConfiguredMinResource(String label, Resource res) {
+    _set(label, ResourceType.MIN_RESOURCE, res);
+  }
+
+  public void setConfiguredMinResource(Resource res) {
+    _set(NL, ResourceType.MIN_RESOURCE, res);
+  }
+
+  /*
+   * Configured Maximum Resource
+   */
+  public Resource getConfiguredMaxResource() {
+    return getConfiguredMaxResource(NL);
+  }
+
+  public Resource getConfiguredMaxResource(String label) {
+    return _get(label, ResourceType.MAX_RESOURCE);
+  }
+
+  public void setConfiguredMaxResource(Resource res) {
+    setConfiguredMaxResource(NL, res);
+  }
+
+  public void setConfiguredMaxResource(String label, Resource res) {
+    _set(label, ResourceType.MAX_RESOURCE, res);
+  }
+
+  /*
+   * Effective Minimum Resource
+   */
+  public Resource getEffectiveMinResource() {
+    return _get(NL, ResourceType.EFF_MIN_RESOURCE);
+  }
+
+  public Resource getEffectiveMinResource(String label) {
+    return _get(label, ResourceType.EFF_MIN_RESOURCE);
+  }
+
+  public void setEffectiveMinResource(String label, Resource res) {
+    _set(label, ResourceType.EFF_MIN_RESOURCE, res);
+  }
+
+  public void setEffectiveMinResource(Resource res) {
+    _set(NL, ResourceType.EFF_MIN_RESOURCE, res);
+  }
+
+  /*
+   * Effective Maximum Resource
+   */
+  public Resource getEffectiveMaxResource() {
+    return getEffectiveMaxResource(NL);
+  }
+
+  public Resource getEffectiveMaxResource(String label) {
+    return _get(label, ResourceType.EFF_MAX_RESOURCE);
+  }
+
+  public void setEffectiveMaxResource(Resource res) {
+    setEffectiveMaxResource(NL, res);
+  }
+
+  public void setEffectiveMaxResource(String label, Resource res) {
+    _set(label, ResourceType.EFF_MAX_RESOURCE, res);
+  }
+
+  /*
+   * Effective Minimum Resource
+   */
+  public Resource getEffectiveMinResourceUp() {
+    return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
+  }
+
+  public Resource getEffectiveMinResourceUp(String label) {
+    return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
+  }
+
+  public void setEffectiveMinResourceUp(String label, Resource res) {
+    _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
+  }
+
+  public void setEffectiveMinResourceUp(Resource res) {
+    _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
+  }
+
+  /*
+   * Effective Maximum Resource
+   */
+  public Resource getEffectiveMaxResourceUp() {
+    return getEffectiveMaxResourceUp(NL);
+  }
+
+  public Resource getEffectiveMaxResourceUp(String label) {
+    return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
+  }
+
+  public void setEffectiveMaxResourceUp(Resource res) {
+    setEffectiveMaxResourceUp(NL, res);
+  }
+
+  public void setEffectiveMaxResourceUp(String label, Resource res) {
+    _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 6f0c7d2..ede4aec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -39,63 +39,12 @@ import org.apache.hadoop.yarn.util.resource.Resources;
  * 
  * And it is thread-safe
  */
-public class ResourceUsage {
-  private ReadLock readLock;
-  private WriteLock writeLock;
-  private Map<String, UsageByLabel> usages;
+public class ResourceUsage extends AbstractResourceUsage {
   // short for no-label :)
   private static final String NL = CommonNodeLabelsManager.NO_LABEL;
-  private final UsageByLabel usageNoLabel;
 
   public ResourceUsage() {
-    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
-    readLock = lock.readLock();
-    writeLock = lock.writeLock();
-
-    usages = new HashMap<String, UsageByLabel>();
-    usageNoLabel = new UsageByLabel(NL);
-    usages.put(NL, usageNoLabel);
-  }
-
-  // Usage enum here to make implement cleaner
-  private enum ResourceType {
-    //CACHED_USED and CACHED_PENDING may be read by anyone, but must only
-    //be written by ordering policies
-    USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
-      CACHED_PENDING(5), AMLIMIT(6);
-
-    private int idx;
-
-    private ResourceType(int value) {
-      this.idx = value;
-    }
-  }
-
-  private static class UsageByLabel {
-    // usage by label, contains all UsageType
-    private Resource[] resArr;
-
-    public UsageByLabel(String label) {
-      resArr = new Resource[ResourceType.values().length];
-      for (int i = 0; i < resArr.length; i++) {
-        resArr[i] = Resource.newInstance(0, 0);
-      };
-    }
-    
-    public Resource getUsed() {
-      return resArr[ResourceType.USED.idx];
-    }
-
-    @Override
-    public String toString() {
-      StringBuilder sb = new StringBuilder();
-      sb.append("{used=" + resArr[0] + "%, ");
-      sb.append("pending=" + resArr[1] + "%, ");
-      sb.append("am_used=" + resArr[2] + "%, ");
-      sb.append("reserved=" + resArr[3] + "%}");
-      sb.append("am_limit=" + resArr[6] + "%, ");
-      return sb.toString();
-    }
+    super();
   }
 
   /*
@@ -109,22 +58,6 @@ public class ResourceUsage {
     return _get(label, ResourceType.USED);
   }
 
-  public Resource getCachedUsed() {
-    return _get(NL, ResourceType.CACHED_USED);
-  }
-
-  public Resource getCachedUsed(String label) {
-    return _get(label, ResourceType.CACHED_USED);
-  }
-
-  public Resource getCachedPending() {
-    return _get(NL, ResourceType.CACHED_PENDING);
-  }
-
-  public Resource getCachedPending(String label) {
-    return _get(label, ResourceType.CACHED_PENDING);
-  }
-
   public void incUsed(String label, Resource res) {
     _inc(label, ResourceType.USED, res);
   }
@@ -145,7 +78,7 @@ public class ResourceUsage {
     setUsed(NL, res);
   }
   
-  public void copyAllUsed(ResourceUsage other) {
+  public void copyAllUsed(AbstractResourceUsage other) {
     try {
       writeLock.lock();
       for (Entry<String, UsageByLabel> entry : other.usages.entrySet()) {
@@ -160,22 +93,6 @@ public class ResourceUsage {
     _set(label, ResourceType.USED, res);
   }
 
-  public void setCachedUsed(String label, Resource res) {
-    _set(label, ResourceType.CACHED_USED, res);
-  }
-
-  public void setCachedUsed(Resource res) {
-    _set(NL, ResourceType.CACHED_USED, res);
-  }
-
-  public void setCachedPending(String label, Resource res) {
-    _set(label, ResourceType.CACHED_PENDING, res);
-  }
-
-  public void setCachedPending(Resource res) {
-    _set(NL, ResourceType.CACHED_PENDING, res);
-  }
-
   /*
    * Pending
    */
@@ -281,6 +198,47 @@ public class ResourceUsage {
     _set(label, ResourceType.AMUSED, res);
   }
 
+  public Resource getAllPending() {
+    return _getAll(ResourceType.PENDING);
+  }
+
+  public Resource getAllUsed() {
+    return _getAll(ResourceType.USED);
+  }
+
+  // Cache Used
+  public Resource getCachedUsed() {
+    return _get(NL, ResourceType.CACHED_USED);
+  }
+
+  public Resource getCachedUsed(String label) {
+    return _get(label, ResourceType.CACHED_USED);
+  }
+
+  public Resource getCachedPending() {
+    return _get(NL, ResourceType.CACHED_PENDING);
+  }
+
+  public Resource getCachedPending(String label) {
+    return _get(label, ResourceType.CACHED_PENDING);
+  }
+
+  public void setCachedUsed(String label, Resource res) {
+    _set(label, ResourceType.CACHED_USED, res);
+  }
+
+  public void setCachedUsed(Resource res) {
+    _set(NL, ResourceType.CACHED_USED, res);
+  }
+
+  public void setCachedPending(String label, Resource res) {
+    _set(label, ResourceType.CACHED_PENDING, res);
+  }
+
+  public void setCachedPending(Resource res) {
+    _set(NL, ResourceType.CACHED_PENDING, res);
+  }
+
   /*
    * AM-Resource Limit
    */
@@ -316,94 +274,6 @@ public class ResourceUsage {
     _set(label, ResourceType.AMLIMIT, res);
   }
 
-  private static Resource normalize(Resource res) {
-    if (res == null) {
-      return Resources.none();
-    }
-    return res;
-  }
-
-  private Resource _get(String label, ResourceType type) {
-    if (label == null || label.equals(NL)) {
-      return normalize(usageNoLabel.resArr[type.idx]);
-    }
-    try {
-      readLock.lock();
-      UsageByLabel usage = usages.get(label);
-      if (null == usage) {
-        return Resources.none();
-      }
-      return normalize(usage.resArr[type.idx]);
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  private Resource _getAll(ResourceType type) {
-    try {
-      readLock.lock();
-      Resource allOfType = Resources.createResource(0);
-      for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
-        //all usages types are initialized
-        Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
-      }
-      return allOfType;
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  public Resource getAllPending() {
-    return _getAll(ResourceType.PENDING);
-  }
-  
-  public Resource getAllUsed() {
-    return _getAll(ResourceType.USED);
-  }
-
-  private UsageByLabel getAndAddIfMissing(String label) {
-    if (label == null || label.equals(NL)) {
-      return usageNoLabel;
-    }
-    if (!usages.containsKey(label)) {
-      UsageByLabel u = new UsageByLabel(label);
-      usages.put(label, u);
-      return u;
-    }
-
-    return usages.get(label);
-  }
-
-  private void _set(String label, ResourceType type, Resource res) {
-    try {
-      writeLock.lock();
-      UsageByLabel usage = getAndAddIfMissing(label);
-      usage.resArr[type.idx] = res;
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void _inc(String label, ResourceType type, Resource res) {
-    try {
-      writeLock.lock();
-      UsageByLabel usage = getAndAddIfMissing(label);
-      Resources.addTo(usage.resArr[type.idx], res);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
-  private void _dec(String label, ResourceType type, Resource res) {
-    try {
-      writeLock.lock();
-      UsageByLabel usage = getAndAddIfMissing(label);
-      Resources.subtractFrom(usage.resArr[type.idx], res);
-    } finally {
-      writeLock.unlock();
-    }
-  }
-
   public Resource getCachedDemand(String label) {
     try {
       readLock.lock();
@@ -415,23 +285,4 @@ public class ResourceUsage {
       readLock.unlock();
     }
   }
-  
-  @Override
-  public String toString() {
-    try {
-      readLock.lock();
-      return usages.toString();
-    } finally {
-      readLock.unlock();
-    }
-  }
-  
-  public Set<String> getNodePartitionsSet() {
-    try {
-      readLock.lock();
-      return usages.keySet();
-    } finally {
-      readLock.unlock();
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 74c85ce..dddac4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -87,6 +89,7 @@ public abstract class AbstractCSQueue implements CSQueue {
 
   final ResourceCalculator resourceCalculator;
   Set<String> accessibleLabels;
+  Set<String> resourceTypes;
   final RMNodeLabelsManager labelManager;
   String defaultLabelExpression;
   
@@ -102,6 +105,14 @@ public abstract class AbstractCSQueue implements CSQueue {
   // etc.
   QueueCapacities queueCapacities;
 
+  QueueResourceQuotas queueResourceQuotas;
+
+  protected enum CapacityConfigType {
+    NONE, PERCENTAGE, ABSOLUTE_RESOURCE
+  };
+  protected CapacityConfigType capacityConfigType =
+      CapacityConfigType.NONE;
+
   private final RecordFactory recordFactory = 
       RecordFactoryProvider.getRecordFactory(null);
   protected CapacitySchedulerContext csContext;
@@ -141,6 +152,9 @@ public abstract class AbstractCSQueue implements CSQueue {
     // initialize QueueCapacities
     queueCapacities = new QueueCapacities(parent == null);
 
+    // initialize queueResourceQuotas
+    queueResourceQuotas = new QueueResourceQuotas();
+
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     readLock = lock.readLock();
     writeLock = lock.writeLock();
@@ -276,6 +290,10 @@ public abstract class AbstractCSQueue implements CSQueue {
       this.defaultLabelExpression =
           csContext.getConfiguration().getDefaultNodeLabelExpression(
               getQueuePath());
+      this.resourceTypes = new HashSet<String>();
+      for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
+        resourceTypes.add(type.toString().toLowerCase());
+      }
 
       // inherit from parent if labels not set
       if (this.accessibleLabels == null && parent != null) {
@@ -292,6 +310,11 @@ public abstract class AbstractCSQueue implements CSQueue {
       // After we setup labels, we can setup capacities
       setupConfigurableCapacities();
 
+      // Also fetch minimum/maximum resource constraint for this queue if
+      // configured.
+      capacityConfigType = CapacityConfigType.NONE;
+      updateConfigurableResourceRequirement(getQueuePath(), clusterResource);
+
       this.maximumAllocation =
           csContext.getConfiguration().getMaximumAllocationPerQueue(
               getQueuePath());
@@ -364,6 +387,125 @@ public abstract class AbstractCSQueue implements CSQueue {
     return unionInheritedWeights;
   }
 
+  protected void updateConfigurableResourceRequirement(String queuePath,
+      Resource clusterResource) {
+    CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+    Set<String> configuredNodelabels = conf.getConfiguredNodeLabels(queuePath);
+
+    for (String label : configuredNodelabels) {
+      Resource minResource = conf.getMinimumResourceRequirement(label,
+          queuePath, resourceTypes);
+      Resource maxResource = conf.getMaximumResourceRequirement(label,
+          queuePath, resourceTypes);
+
+      if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
+        this.capacityConfigType = (!minResource.equals(Resources.none())
+            && queueCapacities.getAbsoluteCapacity(label) == 0f)
+                ? CapacityConfigType.ABSOLUTE_RESOURCE
+                : CapacityConfigType.PERCENTAGE;
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("capacityConfigType is updated as '" + capacityConfigType
+              + "' for queue '" + getQueueName());
+        }
+      }
+
+      validateAbsoluteVsPercentageCapacityConfig(minResource);
+
+      // If min resource for a resource type is greater than its max resource,
+      // throw exception to handle such error configs.
+      if (!maxResource.equals(Resources.none()) && Resources.greaterThan(
+          resourceCalculator, clusterResource, minResource, maxResource)) {
+        throw new IllegalArgumentException("Min resource configuration "
+            + minResource + " is greater than its max value:" + maxResource
+            + " in queue:" + getQueueName());
+      }
+
+      // If parent's max resource is lesser to a specific child's max
+      // resource, throw exception to handle such error configs.
+      if (parent != null) {
+        Resource parentMaxRes = parent.getQueueResourceQuotas()
+            .getConfiguredMaxResource(label);
+        if (Resources.greaterThan(resourceCalculator, clusterResource,
+            parentMaxRes, Resources.none())) {
+          if (Resources.greaterThan(resourceCalculator, clusterResource,
+              maxResource, parentMaxRes)) {
+            throw new IllegalArgumentException("Max resource configuration "
+                + maxResource + " is greater than parents max value:"
+                + parentMaxRes + " in queue:" + getQueueName());
+          }
+
+          // If child's max resource is not set, but its parent max resource is
+          // set, we must set child max resource to its parent's.
+          if (maxResource.equals(Resources.none())
+              && !minResource.equals(Resources.none())) {
+            maxResource = Resources.clone(parentMaxRes);
+          }
+        }
+      }
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Updating absolute resource configuration for queue:"
+            + getQueueName() + " as minResource=" + minResource
+            + " and maxResource=" + maxResource);
+      }
+
+      queueResourceQuotas.setConfiguredMinResource(label, minResource);
+      queueResourceQuotas.setConfiguredMaxResource(label, maxResource);
+    }
+  }
+
+  private void validateAbsoluteVsPercentageCapacityConfig(
+      Resource minResource) {
+    CapacityConfigType localType = CapacityConfigType.PERCENTAGE;
+    if (!minResource.equals(Resources.none())) {
+      localType = CapacityConfigType.ABSOLUTE_RESOURCE;
+    }
+
+    if (!queueName.equals("root")
+        && !this.capacityConfigType.equals(localType)) {
+      throw new IllegalArgumentException("Queue '" + getQueueName()
+          + "' should use either percentage based capacity"
+          + " configuration or absolute resource.");
+    }
+  }
+
+  @Override
+  public CapacityConfigType getCapacityConfigType() {
+    return capacityConfigType;
+  }
+
+  @Override
+  public Resource getEffectiveCapacity(String label) {
+    return Resources
+        .clone(getQueueResourceQuotas().getEffectiveMinResource(label));
+  }
+
+  @Override
+  public Resource getEffectiveCapacityUp(String label) {
+    return Resources
+        .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
+  }
+
+  @Override
+  public Resource getEffectiveCapacityDown(String label, Resource factor) {
+    return Resources.normalizeDown(resourceCalculator,
+        getQueueResourceQuotas().getEffectiveMinResource(label),
+        minimumAllocation);
+  }
+
+  @Override
+  public Resource getEffectiveMaxCapacity(String label) {
+    return Resources
+        .clone(getQueueResourceQuotas().getEffectiveMaxResource(label));
+  }
+
+  @Override
+  public Resource getEffectiveMaxCapacityDown(String label, Resource factor) {
+    return Resources.normalizeDown(resourceCalculator,
+        getQueueResourceQuotas().getEffectiveMaxResource(label),
+        minimumAllocation);
+  }
+
   private void initializeQueueState(QueueState previousState,
       QueueState configuredState, QueueState parentState) {
     // verify that we can not any value for State other than RUNNING/STOPPED
@@ -555,6 +697,11 @@ public abstract class AbstractCSQueue implements CSQueue {
   }
 
   @Override
+  public QueueResourceQuotas getQueueResourceQuotas() {
+    return queueResourceQuotas;
+  }
+
+  @Override
   public ReentrantReadWriteLock.ReadLock getReadLock() {
     return readLock;
   }
@@ -604,7 +751,7 @@ public abstract class AbstractCSQueue implements CSQueue {
        * limit-set-by-parent)
        */
       Resource queueMaxResource =
-          getQueueMaxResource(nodePartition, clusterResource);
+          getQueueMaxResource(nodePartition);
 
       return Resources.min(resourceCalculator, clusterResource,
           queueMaxResource, currentResourceLimits.getLimit());
@@ -617,11 +764,8 @@ public abstract class AbstractCSQueue implements CSQueue {
     return Resources.none();
   }
 
-  Resource getQueueMaxResource(String nodePartition, Resource clusterResource) {
-    return Resources.multiplyAndNormalizeDown(resourceCalculator,
-        labelManager.getResourceByLabel(nodePartition, clusterResource),
-        queueCapacities.getAbsoluteMaximumCapacity(nodePartition),
-        minimumAllocation);
+  Resource getQueueMaxResource(String nodePartition) {
+    return getEffectiveMaxCapacity(nodePartition);
   }
 
   public boolean hasChildQueues() {
@@ -782,7 +926,7 @@ public abstract class AbstractCSQueue implements CSQueue {
     queueUsage.incUsed(nodeLabel, resourceToInc);
     CSQueueUtils.updateUsedCapacity(resourceCalculator,
         labelManager.getResourceByLabel(nodeLabel, Resources.none()),
-        nodeLabel, this);
+        Resources.none(), nodeLabel, this);
     if (null != parent) {
       parent.incUsedResource(nodeLabel, resourceToInc, null);
     }
@@ -798,7 +942,7 @@ public abstract class AbstractCSQueue implements CSQueue {
     queueUsage.decUsed(nodeLabel, resourceToDec);
     CSQueueUtils.updateUsedCapacity(resourceCalculator,
         labelManager.getResourceByLabel(nodeLabel, Resources.none()),
-        nodeLabel, this);
+        Resources.none(), nodeLabel, this);
     if (null != parent) {
       parent.decUsedResource(nodeLabel, resourceToDec, null);
     }
@@ -904,7 +1048,7 @@ public abstract class AbstractCSQueue implements CSQueue {
         Resource maxResourceLimit;
         if (allocation.getSchedulingMode()
             == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
-          maxResourceLimit = getQueueMaxResource(partition, cluster);
+          maxResourceLimit = getQueueMaxResource(partition);
         } else{
           maxResourceLimit = labelManager.getResourceByLabel(
               schedulerContainer.getNodePartition(), cluster);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 43e7f53..2e29a71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -41,10 +41,12 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -355,4 +357,41 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    * @return map of usernames and corresponding weight
    */
   Map<String, Float> getUserWeights();
+
+  /**
+   * Get QueueResourceQuotas associated with each queue.
+   * @return QueueResourceQuotas
+   */
+  public QueueResourceQuotas getQueueResourceQuotas();
+
+  /**
+   * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+   * @return CapacityConfigType
+   */
+  public CapacityConfigType getCapacityConfigType();
+
+  /**
+   * Get effective capacity of queue. If min/max resource is configured,
+   * preference will be given to absolute configuration over normal capacity.
+   * Also round down the result to normalizeDown.
+   *
+   * @param label
+   *          partition
+   * @return effective queue capacity
+   */
+  Resource getEffectiveCapacity(String label);
+  Resource getEffectiveCapacityUp(String label);
+  Resource getEffectiveCapacityDown(String label, Resource factor);
+
+  /**
+   * Get effective max capacity of queue. If min/max resource is configured,
+   * preference will be given to absolute configuration over normal capacity.
+   * Also round down the result to normalizeDown.
+   *
+   * @param label
+   *          partition
+   * @return effective max queue capacity
+   */
+  Resource getEffectiveMaxCapacity(String label);
+  Resource getEffectiveMaxCapacityDown(String label, Resource factor);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e1014c1..81dec80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -150,7 +150,7 @@ class CSQueueUtils {
       }
     }
   }
-  
+
   // Set absolute capacities for {capacity, maximum-capacity}
   private static void updateAbsoluteCapacitiesByNodeLabels(
       QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
@@ -180,8 +180,8 @@ class CSQueueUtils {
    * used resource for all partitions of this queue.
    */
   public static void updateUsedCapacity(final ResourceCalculator rc,
-      final Resource totalPartitionResource, String nodePartition,
-      AbstractCSQueue childQueue) {
+      final Resource totalPartitionResource, Resource clusterResource,
+      String nodePartition, AbstractCSQueue childQueue) {
     QueueCapacities queueCapacities = childQueue.getQueueCapacities();
     CSQueueMetrics queueMetrics = childQueue.getMetrics();
     ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -193,11 +193,8 @@ class CSQueueUtils {
 
     if (Resources.greaterThan(rc, totalPartitionResource,
         totalPartitionResource, Resources.none())) {
-      // queueGuaranteed = totalPartitionedResource *
-      // absolute_capacity(partition)
-      Resource queueGuranteedResource =
-          Resources.multiply(totalPartitionResource,
-              queueCapacities.getAbsoluteCapacity(nodePartition));
+      Resource queueGuranteedResource = childQueue
+          .getEffectiveCapacity(nodePartition);
 
       // make queueGuranteed >= minimum_allocation to avoid divided by 0.
       queueGuranteedResource =
@@ -248,9 +245,7 @@ class CSQueueUtils {
     for (String partition : nodeLabels) {
       // Calculate guaranteed resource for a label in a queue by below logic.
       // (total label resource) * (absolute capacity of label in that queue)
-      Resource queueGuranteedResource = Resources.multiply(nlm
-          .getResourceByLabel(partition, cluster), queue.getQueueCapacities()
-          .getAbsoluteCapacity(partition));
+      Resource queueGuranteedResource = queue.getEffectiveCapacity(partition);
 
       // Available resource in queue for a specific label will be calculated as
       // {(guaranteed resource for a label in a queue) -
@@ -289,15 +284,14 @@ class CSQueueUtils {
     ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
 
     if (nodePartition == null) {
-      for (String partition : Sets.union(
-          queueCapacities.getNodePartitionsSet(),
+      for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
           queueResourceUsage.getNodePartitionsSet())) {
         updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
-            partition, childQueue);
+            cluster, partition, childQueue);
       }
     } else {
       updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
-          nodePartition, childQueue);
+          cluster, nodePartition, childQueue);
     }
 
     // Update queue metrics w.r.t node labels. In a generic way, we can

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 4515453..f81e097 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderi
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -60,6 +61,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.Set;
 import java.util.StringTokenizer;
 
@@ -317,6 +320,21 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   @Private
   public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
 
+  /** Configuring absolute min/max resources in a queue **/
+  @Private
+  public static final String MINIMUM_RESOURCE = "min-resource";
+
+  @Private
+  public static final String MAXIMUM_RESOURCE = "max-resource";
+
+  public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
+
+  public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+
+  public enum AbsoluteResourceType {
+    MEMORY, VCORES;
+  }
+
   AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
 
   public CapacitySchedulerConfiguration() {
@@ -394,7 +412,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   
   public float getNonLabeledQueueCapacity(String queue) {
     float capacity = queue.equals("root") ? 100.0f : getFloat(
-        getQueuePrefix(queue) + CAPACITY, UNDEFINED);
+        getQueuePrefix(queue) + CAPACITY, 0f);
     if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
       throw new IllegalArgumentException("Illegal " +
       		"capacity of " + capacity + " for queue " + queue);
@@ -1676,4 +1694,163 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
         queuePath);
     setMaximumCapacity(leafQueueConfPrefix, val);
   }
+
+  public static String getUnits(String resourceValue) {
+    String units;
+    for (int i = 0; i < resourceValue.length(); i++) {
+      if (Character.isAlphabetic(resourceValue.charAt(i))) {
+        units = resourceValue.substring(i);
+        if (StringUtils.isAlpha(units)) {
+          return units;
+        }
+      }
+    }
+    return "";
+  }
+
+  /**
+   * Get absolute minimum resource requirement for a queue.
+   *
+   * @param label
+   *          NodeLabel
+   * @param queue
+   *          queue path
+   * @param resourceTypes
+   *          Resource types
+   * @return ResourceInformation
+   */
+  public Resource getMinimumResourceRequirement(String label, String queue,
+      Set<String> resourceTypes) {
+    return internalGetLabeledResourceRequirementForQueue(queue, label,
+        resourceTypes, MINIMUM_RESOURCE);
+  }
+
+  /**
+   * Get absolute maximum resource requirement for a queue.
+   *
+   * @param label
+   *          NodeLabel
+   * @param queue
+   *          queue path
+   * @param resourceTypes
+   *          Resource types
+   * @return Resource
+   */
+  public Resource getMaximumResourceRequirement(String label, String queue,
+      Set<String> resourceTypes) {
+    return internalGetLabeledResourceRequirementForQueue(queue, label,
+        resourceTypes, MAXIMUM_RESOURCE);
+  }
+
+  @VisibleForTesting
+  public void setMinimumResourceRequirement(String label, String queue,
+      Resource resource) {
+    updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+  }
+
+  @VisibleForTesting
+  public void setMaximumResourceRequirement(String label, String queue,
+      Resource resource) {
+    updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+  }
+
+  private void updateMinMaxResourceToConf(String label, String queue,
+      Resource resource, String type) {
+    if (queue.equals("root")) {
+      throw new IllegalArgumentException(
+          "Cannot set resource, root queue will take 100% of cluster capacity");
+    }
+
+    StringBuilder resourceString = new StringBuilder();
+    resourceString
+        .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
+            + resource.getMemorySize() + ","
+            + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
+            + resource.getVirtualCores() + "]");
+
+    String prefix = getQueuePrefix(queue) + type;
+    if (!label.isEmpty()) {
+      prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+          + DOT + type;
+    }
+    set(prefix, resourceString.toString());
+  }
+
+  private Resource internalGetLabeledResourceRequirementForQueue(String queue,
+      String label, Set<String> resourceTypes, String suffix) {
+    String propertyName = getNodeLabelPrefix(queue, label) + suffix;
+    String resourceString = get(propertyName);
+    if (resourceString == null || resourceString.isEmpty()) {
+      return Resources.none();
+    }
+
+    // Define resource here.
+    Resource resource = Resource.newInstance(0l, 0);
+    Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
+        .matcher(resourceString);
+    /*
+     * Absolute resource configuration for a queue will be grouped by "[]".
+     * Syntax of absolute resource config could be like below
+     * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores".
+     */
+    if (matcher.find()) {
+      // Get the sub-group.
+      String subGroup = matcher.group(1);
+      if (subGroup.trim().isEmpty()) {
+        return Resources.none();
+      }
+
+      for (String kvPair : subGroup.trim().split(",")) {
+        String[] splits = kvPair.split("=");
+
+        // Ensure that each sub string is key value pair separated by '='.
+        if (splits != null && splits.length > 1) {
+          updateResourceValuesFromConfig(resourceTypes, resource, splits);
+        }
+      }
+    }
+
+    // Memory has to be configured always.
+    if (resource.getMemorySize() == 0l) {
+      return Resources.none();
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("CSConf - getAbsolueResourcePerQueue: prefix="
+          + getNodeLabelPrefix(queue, label) + ", capacity=" + resource);
+    }
+    return resource;
+  }
+
+  private void updateResourceValuesFromConfig(Set<String> resourceTypes,
+      Resource resource, String[] splits) {
+
+    // If key is not a valid type, skip it.
+    if (!resourceTypes.contains(splits[0])) {
+      return;
+    }
+
+    String units = getUnits(splits[1]);
+    Long resourceValue = Long
+        .valueOf(splits[1].substring(0, splits[1].length() - units.length()));
+
+    // Convert all incoming units to MB if units is configured.
+    if (!units.isEmpty()) {
+      resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
+    }
+
+    // map it based on key.
+    AbsoluteResourceType resType = AbsoluteResourceType
+        .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+    switch (resType) {
+      case MEMORY :
+        resource.setMemorySize(resourceValue);
+        break;
+      case VCORES :
+        resource.setVirtualCores(resourceValue.intValue());
+        break;
+      default :
+        break;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index e8342d9..1460121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,12 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
           1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
       effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
 
-      Resource queuePartitionResource = Resources
-          .multiplyAndNormalizeUp(resourceCalculator,
-              labelManager.getResourceByLabel(nodePartition,
-                  lastClusterResource),
-              queueCapacities.getAbsoluteCapacity(nodePartition),
-              minimumAllocation);
+      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
 
       Resource userAMLimit = Resources.multiplyAndNormalizeUp(
           resourceCalculator, queuePartitionResource,
@@ -705,11 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
        * non-labeled), * with per-partition am-resource-percent to get the max am
        * resource limit for this queue and partition.
        */
-      Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
-          resourceCalculator,
-          labelManager.getResourceByLabel(nodePartition, lastClusterResource),
-          queueCapacities.getAbsoluteCapacity(nodePartition),
-          minimumAllocation);
+      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
 
       Resource queueCurrentLimit = Resources.none();
       // For non-labeled partition, we need to consider the current queue
@@ -965,6 +956,14 @@ public class LeafQueue extends AbstractCSQueue {
   private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
     // Set preemption-allowed:
     // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
+    if (!queueResourceQuotas.getEffectiveMinResource(nodePartition)
+        .equals(Resources.none())) {
+      limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
+          csContext.getClusterResource(), queueUsage.getUsed(nodePartition),
+          queueResourceQuotas.getEffectiveMinResource(nodePartition)));
+      return;
+    }
+
     float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
     float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
     limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
@@ -1344,7 +1343,7 @@ public class LeafQueue extends AbstractCSQueue {
     currentPartitionResourceLimit =
         partition.equals(RMNodeLabelsManager.NO_LABEL)
             ? currentPartitionResourceLimit
-            : getQueueMaxResource(partition, clusterResource);
+            : getQueueMaxResource(partition);
 
     Resource headroom = Resources.componentwiseMin(
         Resources.subtract(userLimitResource, user.getUsed(partition)),
@@ -1716,12 +1715,8 @@ public class LeafQueue extends AbstractCSQueue {
     // this. So need cap limits by queue's max capacity here.
     this.cachedResourceLimitsForHeadroom =
         new ResourceLimits(currentResourceLimits.getLimit());
-    Resource queueMaxResource =
-        Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
-            .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
-            queueCapacities
-                .getAbsoluteMaximumCapacity(RMNodeLabelsManager.NO_LABEL),
-            minimumAllocation);
+    Resource queueMaxResource = getEffectiveMaxCapacityDown(
+        RMNodeLabelsManager.NO_LABEL, minimumAllocation);
     this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
         resourceCalculator, clusterResource, queueMaxResource,
         currentResourceLimits.getLimit()));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 959ca51..c770fac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 @Private
@@ -162,31 +164,78 @@ public class ParentQueue extends AbstractCSQueue {
       writeLock.lock();
       // Validate
       float childCapacities = 0;
+      Resource minResDefaultLabel = Resources.createResource(0, 0);
       for (CSQueue queue : childQueues) {
         childCapacities += queue.getCapacity();
+        Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas()
+            .getConfiguredMinResource());
+
+        // If any child queue is using percentage based capacity model vs parent
+        // queues' absolute configuration or vice versa, throw back an
+        // exception.
+        if (!queueName.equals("root") && getCapacity() != 0f
+            && !queue.getQueueResourceQuotas().getConfiguredMinResource()
+                .equals(Resources.none())) {
+          throw new IllegalArgumentException("Parent queue '" + getQueueName()
+              + "' and child queue '" + queue.getQueueName()
+              + "' should use either percentage based capacity"
+              + " configuration or absolute resource together.");
+        }
       }
+
       float delta = Math.abs(1.0f - childCapacities);  // crude way to check
       // allow capacities being set to 0, and enforce child 0 if parent is 0
-      if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
-          (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
-        throw new IllegalArgumentException(
-            "Illegal" + " capacity of " + childCapacities
-                + " for children of queue " + queueName);
+      if ((minResDefaultLabel.equals(Resources.none())
+          && (queueCapacities.getCapacity() > 0) && (delta > PRECISION))
+          || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+        throw new IllegalArgumentException("Illegal" + " capacity of "
+            + childCapacities + " for children of queue " + queueName);
       }
       // check label capacities
       for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
         float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
         // check children's labels
         float sum = 0;
+        Resource minRes = Resources.createResource(0, 0);
+        Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+            scheduler.getClusterResource());
         for (CSQueue queue : childQueues) {
           sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+
+          // If any child queue of a label is using percentage based capacity
+          // model vs parent queues' absolute configuration or vice versa, throw
+          // back an exception
+          if (!queueName.equals("root") && !this.capacityConfigType
+              .equals(queue.getCapacityConfigType())) {
+            throw new IllegalArgumentException("Parent queue '" + getQueueName()
+                + "' and child queue '" + queue.getQueueName()
+                + "' should use either percentage based capacity"
+                + "configuration or absolute resource together for label:"
+                + nodeLabel);
+          }
+
+          // Accumulate all min/max resource configured for all child queues.
+          Resources.addTo(minRes, queue.getQueueResourceQuotas()
+              .getConfiguredMinResource(nodeLabel));
         }
-        if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+        if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0
+            && Math.abs(1.0f - sum) > PRECISION)
             || (capacityByLabel == 0) && (sum > 0)) {
           throw new IllegalArgumentException(
               "Illegal" + " capacity of " + sum + " for children of queue "
                   + queueName + " for label=" + nodeLabel);
         }
+
+        // Ensure that for each parent queue: parent.min-resource >=
+        // Σ(child.min-resource).
+        Resource parentMinResource = queueResourceQuotas
+            .getConfiguredMinResource(nodeLabel);
+        if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
+            resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
+          throw new IllegalArgumentException("Parent Queues" + " capacity: "
+              + parentMinResource + " is less than" + " to its children:"
+              + minRes + " for queue:" + queueName);
+        }
       }
 
       this.childQueues.clear();
@@ -687,11 +736,8 @@ public class ParentQueue extends AbstractCSQueue {
         child.getQueueResourceUsage().getUsed(nodePartition));
 
     // Get child's max resource
-    Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(
-        resourceCalculator,
-        labelManager.getResourceByLabel(nodePartition, clusterResource),
-        child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition),
-        minimumAllocation);
+    Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
+        nodePartition, minimumAllocation);
 
     // Child's limit should be capped by child configured max resource
     childLimit =
@@ -827,6 +873,14 @@ public class ParentQueue extends AbstractCSQueue {
       ResourceLimits resourceLimits) {
     try {
       writeLock.lock();
+
+      // Update effective capacity in all parent queue.
+      Set<String> configuredNodelabels = csContext.getConfiguration()
+          .getConfiguredNodeLabels(getQueuePath());
+      for (String label : configuredNodelabels) {
+        calculateEffectiveResourcesAndCapacity(label, clusterResource);
+      }
+
       // Update all children
       for (CSQueue childQueue : childQueues) {
         // Get ResourceLimits of child queue before assign containers
@@ -848,6 +902,110 @@ public class ParentQueue extends AbstractCSQueue {
     return true;
   }
 
+  private void calculateEffectiveResourcesAndCapacity(String label,
+      Resource clusterResource) {
+
+    // For root queue, ensure that max/min resource is updated to latest
+    // cluster resource.
+    Resource resourceByLabel = labelManager.getResourceByLabel(label,
+        clusterResource);
+    if (getQueueName().equals("root")) {
+      queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel);
+      queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
+      queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
+      queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+    }
+
+    // Total configured min resources of direct children of queue
+    Resource configuredMinResources = Resource.newInstance(0L, 0);
+    for (CSQueue childQueue : getChildQueues()) {
+      Resources.addTo(configuredMinResources,
+          childQueue.getQueueResourceQuotas().getConfiguredMinResource(label));
+    }
+
+    // Factor to scale down effective resource: When cluster has sufficient
+    // resources, effective_min_resources will be same as configured
+    // min_resources.
+    float effectiveMinRatio = 1;
+    ResourceCalculator rc = this.csContext.getResourceCalculator();
+    if (getQueueName().equals("root")) {
+      if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
+          clusterResource, resourceByLabel, configuredMinResources)) {
+        effectiveMinRatio = Resources.divide(rc, clusterResource,
+            resourceByLabel, configuredMinResources);
+      }
+    } else {
+      if (Resources.lessThan(rc, clusterResource,
+          queueResourceQuotas.getEffectiveMinResource(label),
+          configuredMinResources)) {
+        effectiveMinRatio = Resources.divide(rc, clusterResource,
+            queueResourceQuotas.getEffectiveMinResource(label),
+            configuredMinResources);
+      }
+    }
+
+    // loop and do this for all child queues
+    for (CSQueue childQueue : getChildQueues()) {
+      Resource minResource = childQueue.getQueueResourceQuotas()
+          .getConfiguredMinResource(label);
+
+      // Update effective resource (min/max) to each child queue.
+      if (childQueue.getCapacityConfigType()
+          .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
+        childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+            Resources.multiply(minResource, effectiveMinRatio));
+
+        // Max resource of a queue should be a minimum of {configuredMaxRes,
+        // parentMaxRes}. parentMaxRes could be configured value. But if not
+        // present could also be taken from effective max resource of parent.
+        Resource parentMaxRes = queueResourceQuotas
+            .getConfiguredMaxResource(label);
+        if (parentMaxRes.equals(Resources.none())) {
+          parentMaxRes = parent.getQueueResourceQuotas()
+              .getEffectiveMaxResource(label);
+        }
+
+        // Minimum of {childMaxResource, parentMaxRes}. However if
+        // childMaxResource is empty, consider parent's max resource alone.
+        Resource childMaxResource = childQueue.getQueueResourceQuotas()
+            .getConfiguredMaxResource(label);
+        Resource effMaxResource = Resources.min(resourceCalculator,
+            resourceByLabel, childMaxResource.equals(Resources.none())
+                ? parentMaxRes
+                : childMaxResource,
+            parentMaxRes);
+        childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+            Resources.clone(effMaxResource));
+      } else {
+        childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+            Resources.multiply(resourceByLabel,
+                childQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+        childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+            Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
+                .getAbsoluteMaximumCapacity(label)));
+
+        childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
+            Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
+                childQueue.getQueueCapacities().getAbsoluteCapacity(label),
+                minimumAllocation));
+        childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
+            Resources.multiplyAndNormalizeUp(rc,
+                resourceByLabel, childQueue.getQueueCapacities()
+                    .getAbsoluteMaximumCapacity(label),
+                    minimumAllocation));
+      }
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Updating effective min resource for queue:"
+            + childQueue.getQueueName() + " as effMinResource="
+            + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label)
+            + "and Updating effective max resource as effMaxResource="
+            + childQueue.getQueueResourceQuotas()
+                .getEffectiveMaxResource(label));
+      }
+    }
+  }
+
   @Override
   public List<CSQueue> getChildQueues() {
     try {
@@ -980,9 +1138,21 @@ public class ParentQueue extends AbstractCSQueue {
        * When this happens, we have to preempt killable container (on same or different
        * nodes) of parent queue to avoid violating parent's max resource.
        */
-      if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
-          < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
-        killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+      if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+          .equals(Resources.none())) {
+        if (Resources.lessThan(resourceCalculator, clusterResource,
+            queueResourceQuotas.getEffectiveMaxResource(nodePartition),
+            queueUsage.getUsed(nodePartition))) {
+          killContainersToEnforceMaxQueueCapacity(nodePartition,
+              clusterResource);
+        }
+      } else {
+        if (getQueueCapacities()
+            .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities()
+                .getAbsoluteUsedCapacity(nodePartition)) {
+          killContainersToEnforceMaxQueueCapacity(nodePartition,
+              clusterResource);
+        }
       }
     } finally {
       writeLock.unlock();
@@ -999,8 +1169,7 @@ public class ParentQueue extends AbstractCSQueue {
 
     Resource partitionResource = labelManager.getResourceByLabel(partition,
         null);
-    Resource maxResource = Resources.multiply(partitionResource,
-        getQueueCapacities().getAbsoluteMaximumCapacity(partition));
+    Resource maxResource = getEffectiveMaxCapacity(partition);
 
     while (Resources.greaterThan(resourceCalculator, partitionResource,
         queueUsage.getUsed(partition), maxResource)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 33f30b0..efc20e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,10 +686,7 @@ public class UsersManager implements AbstractUsersManager {
      * * If we're running over capacity, then its (usedResources + required)
      * (which extra resources we are allocating)
      */
-    Resource queueCapacity = Resources.multiplyAndNormalizeUp(
-        resourceCalculator, partitionResource,
-        lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition),
-        lQueue.getMinimumAllocation());
+    Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
 
     /*
      * Assume we have required resource equals to minimumAllocation, this can


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[18/50] hadoop git commit: YARN-7411. Inter-Queue preemption's computeFixpointAllocation need to handle absolute resources while computing normalizedGuarantee. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7411. Inter-Queue preemption's computeFixpointAllocation need to handle absolute resources while computing normalizedGuarantee. (Sunil G via wangda)

Change-Id: I41b1d7558c20fc4eb2050d40134175a2ef6330cb


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/034b312d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/034b312d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/034b312d

Branch: refs/heads/HDFS-7240
Commit: 034b312d9f19024d2eabd377210d17d4080ef70e
Parents: aa3f627
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 13 16:26:27 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../api/records/impl/pb/ResourcePBImpl.java     | 12 ----
 .../resource/DefaultResourceCalculator.java     |  8 +++
 .../resource/DominantResourceCalculator.java    | 21 ++++++
 .../yarn/util/resource/ResourceCalculator.java  | 14 +++-
 .../hadoop/yarn/util/resource/Resources.java    |  5 ++
 .../AbstractPreemptableResourceCalculator.java  | 24 ++++++-
 .../monitor/capacity/TempQueuePerPartition.java | 12 ++--
 ...alCapacityPreemptionPolicyMockFramework.java | 14 ++++
 ...pacityPreemptionPolicyForNodePartitions.java | 76 ++++++++++++++++++++
 9 files changed, 166 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 401e0c0..4f90133 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
@@ -152,17 +151,6 @@ public class ResourcePBImpl extends Resource {
         .newInstance(ResourceInformation.VCORES);
     this.setMemorySize(p.getMemory());
     this.setVirtualCores(p.getVirtualCores());
-
-    // Update missing resource information on respective index.
-    updateResourceInformationMap(types);
-  }
-
-  private void updateResourceInformationMap(ResourceInformation[] types) {
-    for (int i = 0; i < types.length; i++) {
-      if (resources[i] == null) {
-        resources[i] = ResourceInformation.newInstance(types[i]);
-      }
-    }
   }
 
   private static ResourceInformation newDefaultInformation(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index aefa85c..6375c4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -112,6 +112,14 @@ public class DefaultResourceCalculator extends ResourceCalculator {
   }
 
   @Override
+  public Resource multiplyAndNormalizeUp(Resource r, double[] by,
+      Resource stepFactor) {
+    return Resources.createResource(
+        roundUp((long) (r.getMemorySize() * by[0] + 0.5),
+            stepFactor.getMemorySize()));
+  }
+
+  @Override
   public Resource multiplyAndNormalizeDown(Resource r, double by,
       Resource stepFactor) {
     return Resources.createResource(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index cc4acf6..26ddc18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -496,6 +496,27 @@ public class DominantResourceCalculator extends ResourceCalculator {
   }
 
   @Override
+  public Resource multiplyAndNormalizeUp(Resource r, double[] by,
+      Resource stepFactor) {
+    Resource ret = Resource.newInstance(r);
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+    for (int i = 0; i < maxLength; i++) {
+      ResourceInformation rResourceInformation = r.getResourceInformation(i);
+      ResourceInformation stepFactorResourceInformation = stepFactor
+          .getResourceInformation(i);
+
+      long rValue = rResourceInformation.getValue();
+      long stepFactorValue = UnitsConversionUtil.convert(
+          stepFactorResourceInformation.getUnits(),
+          rResourceInformation.getUnits(),
+          stepFactorResourceInformation.getValue());
+      ret.setResourceValue(i, ResourceCalculator
+          .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue));
+    }
+    return ret;
+  }
+  
+  @Override
   public Resource multiplyAndNormalizeUp(Resource r, double by,
       Resource stepFactor) {
     return this.multiplyAndNormalize(r, by, stepFactor, true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index a816290..dfd6684 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -125,7 +125,19 @@ public abstract class ResourceCalculator {
    */
   public abstract Resource multiplyAndNormalizeUp(
       Resource r, double by, Resource stepFactor);
-  
+
+  /**
+   * Multiply resource <code>r</code> by factor <code>by</code> 
+   * and normalize up using step-factor <code>stepFactor</code>.
+   * 
+   * @param r resource to be multiplied
+   * @param by multiplier array for all resource types
+   * @param stepFactor factor by which to normalize up 
+   * @return resulting normalized resource
+   */
+  public abstract Resource multiplyAndNormalizeUp(
+      Resource r, double[] by, Resource stepFactor);
+
   /**
    * Multiply resource <code>r</code> by factor <code>by</code> 
    * and normalize down using step-factor <code>stepFactor</code>.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 0426ab7..1c08844 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -347,6 +347,11 @@ public class Resources {
     return lhs;
   }
 
+  public static Resource multiplyAndNormalizeUp(ResourceCalculator calculator,
+      Resource lhs, double[] by, Resource factor) {
+    return calculator.multiplyAndNormalizeUp(lhs, by, factor);
+  }
+
   public static Resource multiplyAndNormalizeUp(
       ResourceCalculator calculator,Resource lhs, double by, Resource factor) {
     return calculator.multiplyAndNormalizeUp(lhs, by, factor);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
index a80f317..5196831 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
@@ -19,8 +19,11 @@
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
@@ -198,18 +201,33 @@ public class AbstractPreemptableResourceCalculator {
   private void resetCapacity(Resource clusterResource,
       Collection<TempQueuePerPartition> queues, boolean ignoreGuar) {
     Resource activeCap = Resource.newInstance(0, 0);
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
 
     if (ignoreGuar) {
       for (TempQueuePerPartition q : queues) {
-        q.normalizedGuarantee = 1.0f / queues.size();
+        for (int i = 0; i < maxLength; i++) {
+          q.normalizedGuarantee[i] = 1.0f / queues.size();
+        }
       }
     } else {
       for (TempQueuePerPartition q : queues) {
         Resources.addTo(activeCap, q.getGuaranteed());
       }
       for (TempQueuePerPartition q : queues) {
-        q.normalizedGuarantee = Resources.divide(rc, clusterResource,
-            q.getGuaranteed(), activeCap);
+        for (int i = 0; i < maxLength; i++) {
+          ResourceInformation nResourceInformation = q.getGuaranteed()
+              .getResourceInformation(i);
+          ResourceInformation dResourceInformation = activeCap
+              .getResourceInformation(i);
+
+          long nValue = nResourceInformation.getValue();
+          long dValue = UnitsConversionUtil.convert(
+              dResourceInformation.getUnits(), nResourceInformation.getUnits(),
+              dResourceInformation.getValue());
+          if (dValue != 0) {
+            q.normalizedGuarantee[i] = (float) nValue / dValue;
+          }
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index bd236fe..4d71223 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -22,9 +22,11 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -46,7 +48,7 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
   Resource untouchableExtra;
   Resource preemptableExtra;
 
-  double normalizedGuarantee;
+  double[] normalizedGuarantee;
 
   private Resource effMinRes;
   private Resource effMaxRes;
@@ -88,7 +90,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
       pendingDeductReserved = Resources.createResource(0);
     }
 
-    this.normalizedGuarantee = Float.NaN;
+    this.normalizedGuarantee = new double[ResourceUtils
+        .getNumberOfKnownResourceTypes()];
     this.children = new ArrayList<>();
     this.apps = new ArrayList<>();
     this.untouchableExtra = Resource.newInstance(0, 0);
@@ -240,8 +243,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
     sb.append(" NAME: " + queueName).append(" CUR: ").append(current)
         .append(" PEN: ").append(pending).append(" RESERVED: ").append(reserved)
         .append(" GAR: ").append(getGuaranteed()).append(" NORM: ")
-        .append(normalizedGuarantee).append(" IDEAL_ASSIGNED: ")
-        .append(idealAssigned).append(" IDEAL_PREEMPT: ").append(toBePreempted)
+        .append(Arrays.toString(normalizedGuarantee))
+        .append(" IDEAL_ASSIGNED: ").append(idealAssigned)
+        .append(" IDEAL_PREEMPT: ").append(toBePreempted)
         .append(" ACTUAL_PREEMPT: ").append(getActuallyToBePreempted())
         .append(" UNTOUCHABLE: ").append(untouchableExtra)
         .append(" PREEMPTABLE: ").append(preemptableExtra).append("\n");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 1dda8a6..ca43a95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Before;
@@ -533,6 +535,18 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
     } else {
       res = Resources.createResource(Integer.valueOf(resource[0]),
           Integer.valueOf(resource[1]));
+      if (resource.length > 2) {
+        // Using the same order of resources from ResourceUtils, set resource
+        // informations.
+        ResourceInformation[] storedResourceInfo = ResourceUtils
+            .getResourceTypesArray();
+        for (int i = 2; i < resource.length; i++) {
+          res.setResourceInformation(storedResourceInfo[i].getName(),
+              ResourceInformation.newInstance(storedResourceInfo[i].getName(),
+                  storedResourceInfo[i].getUnits(),
+                  Integer.valueOf(resource[i])));
+        }
+      }
     }
     return res;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/034b312d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index 1fd455a..67c09cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -18,11 +18,17 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.mockito.Matchers.argThat;
 import static org.mockito.Mockito.never;
@@ -613,4 +619,74 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions
     verify(mDisp, never()).handle(
         argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
   }
+
+  @Test
+  public void testNormalizeGuaranteeWithMultipleResource() throws IOException {
+    // Initialize resource map
+    Map<String, ResourceInformation> riMap = new HashMap<>();
+    String RESOURCE_1 = "res1";
+
+    // Initialize mandatory resources
+    ResourceInformation memory = ResourceInformation.newInstance(
+        ResourceInformation.MEMORY_MB.getName(),
+        ResourceInformation.MEMORY_MB.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    ResourceInformation vcores = ResourceInformation.newInstance(
+        ResourceInformation.VCORES.getName(),
+        ResourceInformation.VCORES.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    riMap.put(ResourceInformation.MEMORY_URI, memory);
+    riMap.put(ResourceInformation.VCORES_URI, vcores);
+    riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+        ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
+
+    ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+    /**
+     * Queue structure is:
+     *
+     * <pre>
+     *           root
+     *           /  \
+     *          a    b
+     *        /  \  /  \
+     *       a1  a2 b1  b2
+     * </pre>
+     *
+     * a1 and b2 are using most of resources.
+     * a2 and b1 needs more resources. Both are under served.
+     * hence demand will consider both queue's need while trying to
+     * do preemption.
+     */
+    String labelsConfig =
+        "=100,true;";
+    String nodesConfig =
+        "n1=;"; // n1 is default partition
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100:100:10 100:100:10 100:100:10 100:100:10]);" + //root
+        "-a(=[50:80:4 100:100:10 80:90:10 30:20:4]);" + // a
+        "--a1(=[25:30:2 100:50:10 80:90:10 0]);" + // a1
+        "--a2(=[25:50:2 100:50:10 0 30:20:4]);" + // a2
+        "-b(=[50:20:6 100:100:10 20:10 40:50:8]);" + // b
+        "--b1(=[25:5:4 100:20:10 0 20:10:4]);" + // b1
+        "--b2(=[25:15:2 100:20:10 20:10 20:10:4])"; // b2
+    String appsConfig=
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a1\t" // app1 in a1
+        + "(1,8:9:1,n1,,10,false);" +
+        "b2\t" // app2 in b2
+        + "(1,2:1,n1,,10,false)"; // 80 of y
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+    policy.editSchedule();
+
+    verify(mDisp, times(7)).handle(
+        argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
+
+    riMap.remove(RESOURCE_1);
+    ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[46/50] hadoop git commit: Queue ACL validations should validate parent queue ACLs before auto-creating leaf queues. Contributed by Suma Shivaprasad.

Posted by xy...@apache.org.
Queue ACL validations should validate parent queue ACLs before auto-creating leaf queues. Contributed by Suma Shivaprasad.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bb83a8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bb83a8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bb83a8f

Branch: refs/heads/HDFS-7240
Commit: 8bb83a8f625953e2a45db4bbbfb95cd41bac6af5
Parents: 5c87fb2
Author: Sunil G <su...@apache.org>
Authored: Tue Dec 12 15:20:59 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Dec 12 15:20:59 2017 +0530

----------------------------------------------------------------------
 .../server/resourcemanager/RMAppManager.java    |  10 +-
 .../server/resourcemanager/TestAppManager.java  | 112 ++++++++++++++++---
 2 files changed, 108 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bb83a8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 3ad2c9a..b21fb73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -410,7 +410,15 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
       String queueName = submissionContext.getQueue();
       String appName = submissionContext.getApplicationName();
       CSQueue csqueue = ((CapacityScheduler) scheduler).getQueue(queueName);
-      if (null != csqueue
+
+      if (csqueue == null && placementContext != null) {
+        //could be an auto created queue through queue mapping. Validate
+        // parent queue exists and has valid acls
+        String parentQueueName = placementContext.getParentQueue();
+        csqueue = ((CapacityScheduler) scheduler).getQueue(parentQueueName);
+      }
+
+      if (csqueue != null
           && !authorizer.checkPermission(
               new AccessRequest(csqueue.getPrivilegedEntity(), userUgi,
                   SchedulerUtils.toAccessType(QueueACL.SUBMIT_APPLICATIONS),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bb83a8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index f56de0f..7c2ce64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.isA;
+import static org.mockito.Matchers.matches;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -87,6 +88,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
     .CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ManagedParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -279,24 +284,30 @@ public class TestAppManager{
     YarnConfiguration conf = new YarnConfiguration();
     conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
         ResourceScheduler.class);
-    conf.set(PREFIX + "root.queues", "default,test");
 
-    conf.setFloat(PREFIX + "root.default.capacity", 50.0f);
-    conf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
+    CapacitySchedulerConfiguration csConf = new
+        CapacitySchedulerConfiguration(conf, false);
+    csConf.set(PREFIX + "root.queues", "default,test");
 
-    conf.setFloat(PREFIX + "root.test.capacity", 50.0f);
-    conf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f);
+    csConf.setFloat(PREFIX + "root.default.capacity", 50.0f);
+    csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
 
-    conf.set(PREFIX + "root.acl_submit_applications", " ");
-    conf.set(PREFIX + "root.acl_administer_queue", " ");
+    csConf.setFloat(PREFIX + "root.test.capacity", 50.0f);
+    csConf.setFloat(PREFIX + "root.test.maximum-capacity", 100.0f);
 
-    conf.set(PREFIX + "root.default.acl_submit_applications", " ");
-    conf.set(PREFIX + "root.default.acl_administer_queue", " ");
+    csConf.set(PREFIX + "root.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.acl_administer_queue", " ");
 
-    conf.set(PREFIX + "root.test.acl_submit_applications", "test");
-    conf.set(PREFIX + "root.test.acl_administer_queue", "test");
+    csConf.set(PREFIX + "root.default.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.default.acl_administer_queue", " ");
 
-    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
+    csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
+    csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
+
+    csConf.set(PREFIX + "root.test.acl_submit_applications", "test");
+    csConf.set(PREFIX + "root.test.acl_administer_queue", "test");
+
+    csConf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
 
     // Setup a PlacementManager returns a new queue
     PlacementManager placementMgr = mock(PlacementManager.class);
@@ -309,7 +320,7 @@ public class TestAppManager{
       }
 
     }).when(placementMgr).placeApplication(
-        any(ApplicationSubmissionContext.class), any(String.class));
+        any(ApplicationSubmissionContext.class), matches("test"));
 
     asContext.setQueue("oldQueue");
 
@@ -331,12 +342,87 @@ public class TestAppManager{
     try {
       //should fail since user does not have permission to submit to queue
       // 'test'
+      asContext.setApplicationId(appId = MockApps.newAppID(2));
       newAppMonitor.submitApplication(asContext, "test1");
     } catch(YarnException e) {
       assertTrue(e.getCause() instanceof AccessControlException);
     }
   }
 
+  @Test
+  public void
+  testQueueSubmitWithACLsEnabledWithQueueMappingForAutoCreatedQueue()
+      throws IOException, YarnException, InterruptedException {
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        conf, false);
+    csConf.set(PREFIX + "root.queues", "default,managedparent");
+
+    csConf.setFloat(PREFIX + "root.default.capacity", 50.0f);
+    csConf.setFloat(PREFIX + "root.default.maximum-capacity", 100.0f);
+
+    csConf.set(PREFIX + "root.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.acl_administer_queue", " ");
+
+    csConf.set(PREFIX + "root.default.acl_submit_applications", " ");
+    csConf.set(PREFIX + "root.default.acl_administer_queue", " ");
+
+    csConf.set(PREFIX + "root.managedparent.acl_administer_queue", "admin");
+    csConf.set(PREFIX + "root.managedparent.acl_submit_applications", "user1");
+
+    csConf.setAutoCreateChildQueueEnabled("root.managedparent", true);
+    csConf.setAutoCreatedLeafQueueConfigCapacity("root.managedparent", 30f);
+    csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.managedparent", 100f);
+
+    // Setup a PlacementManager returns a new queue
+    PlacementManager placementMgr = mock(PlacementManager.class);
+    doAnswer(new Answer<ApplicationPlacementContext>() {
+
+      @Override
+      public ApplicationPlacementContext answer(InvocationOnMock invocation)
+          throws Throwable {
+        return new ApplicationPlacementContext("user1", "managedparent");
+      }
+
+    }).when(placementMgr).placeApplication(
+        any(ApplicationSubmissionContext.class), matches("user1|user2"));
+
+    asContext.setQueue("oldQueue");
+
+    MockRM newMockRM = new MockRM(conf);
+    CapacityScheduler cs =
+        ((CapacityScheduler) newMockRM.getResourceScheduler());
+    ManagedParentQueue managedParentQueue = new ManagedParentQueue(cs,
+        "managedparent", cs.getQueue("root"), null);
+    cs.getCapacitySchedulerQueueManager().addQueue("managedparent",
+        managedParentQueue);
+
+    RMContext newMockRMContext = newMockRM.getRMContext();
+    newMockRMContext.setQueuePlacementManager(placementMgr);
+    ApplicationMasterService masterService = new ApplicationMasterService(
+        newMockRMContext, newMockRMContext.getScheduler());
+
+    TestRMAppManager newAppMonitor = new TestRMAppManager(newMockRMContext,
+        new ClientToAMTokenSecretManagerInRM(), newMockRMContext.getScheduler(),
+        masterService, new ApplicationACLsManager(conf), conf);
+
+    //only user test has permission to submit to 'test' queue
+    newAppMonitor.submitApplication(asContext, "user1");
+
+    try {
+      //should fail since user does not have permission to submit to queue
+      // 'test'
+      asContext.setApplicationId(appId = MockApps.newAppID(2));
+      newAppMonitor.submitApplication(asContext, "user2");
+    } catch (YarnException e) {
+      assertTrue(e.getCause() instanceof AccessControlException);
+    }
+  }
+
   @After
   public void tearDown() {
     setAppEventType(RMAppEventType.KILL);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[20/50] hadoop git commit: YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d52627a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d52627a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d52627a7

Branch: refs/heads/HDFS-7240
Commit: d52627a7cbddfd981db973e223aefffde1ebf82d
Parents: 5e798b1
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 16 11:04:05 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../yarn/api/records/QueueConfigurations.java   |  76 +++++++++++
 .../yarn/api/records/ResourceInformation.java   |   6 +
 .../src/main/proto/yarn_protos.proto            |   4 +
 .../impl/pb/QueueConfigurationsPBImpl.java      | 136 ++++++++++++++++++-
 .../resource/DominantResourceCalculator.java    |  24 +++-
 .../scheduler/QueueResourceQuotas.java          |  38 ------
 .../scheduler/capacity/AbstractCSQueue.java     |  18 ++-
 .../scheduler/capacity/CSQueue.java             |   4 +-
 .../scheduler/capacity/LeafQueue.java           |   4 +-
 .../scheduler/capacity/ParentQueue.java         |  56 ++++++--
 .../scheduler/capacity/UsersManager.java        |   2 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java |  23 +++-
 .../webapp/CapacitySchedulerPage.java           |  49 +++++--
 .../webapp/dao/CapacitySchedulerInfo.java       |   3 +-
 .../dao/CapacitySchedulerLeafQueueInfo.java     |   6 +-
 .../webapp/dao/CapacitySchedulerQueueInfo.java  |  10 +-
 .../dao/PartitionQueueCapacitiesInfo.java       |  34 ++++-
 .../webapp/dao/QueueCapacitiesInfo.java         |  13 +-
 ...CapacitySchedulerWithMultiResourceTypes.java |  25 ++--
 .../scheduler/capacity/TestLeafQueue.java       |  24 +++-
 20 files changed, 443 insertions(+), 112 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
index e25c8aa..30096bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
@@ -147,4 +147,80 @@ public abstract class QueueConfigurations {
   @Private
   @Unstable
   public abstract void setMaxAMPercentage(float maxAMPercentage);
+
+  /**
+   * Get the effective minimum capacity of queue (from absolute resource).
+   *
+   * @return minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getEffectiveMinCapacity();
+
+  /**
+   * Set the effective minimum capacity of queue (from absolute resource).
+   *
+   * @param capacity
+   *          minimum resource capability
+   */
+  @Private
+  @Unstable
+  public abstract void setEffectiveMinCapacity(Resource capacity);
+
+  /**
+   * Get the effective maximum capacity of queue (from absolute resource).
+   *
+   * @return maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getEffectiveMaxCapacity();
+
+  /**
+   * Set the effective maximum capacity of queue (from absolute resource).
+   *
+   * @param capacity
+   *          maximum resource capability
+   */
+  @Private
+  @Unstable
+  public abstract void setEffectiveMaxCapacity(Resource capacity);
+
+  /**
+   * Get the configured minimum capacity of queue (from absolute resource).
+   *
+   * @return minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getConfiguredMinCapacity();
+
+  /**
+   * Set the configured minimum capacity of queue (from absolute resource).
+   *
+   * @param configuredMinResource
+   *          minimum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract void setConfiguredMinCapacity(Resource configuredMinResource);
+
+  /**
+   * Get the configured maximum capacity of queue (from absolute resource).
+   *
+   * @return maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract Resource getConfiguredMaxCapacity();
+
+  /**
+   * Set the configured maximum capacity of queue (from absolute resource).
+   *
+   * @param configuredMaxResource
+   *          maximum resource capability
+   */
+  @Public
+  @Unstable
+  public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index a8198d8..904ff4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -228,6 +228,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
         Long.MAX_VALUE);
   }
 
+  public static ResourceInformation newInstance(String name, String units,
+      long minRes, long maxRes) {
+    return ResourceInformation.newInstance(name, units, 0L,
+        ResourceTypes.COUNTABLE, minRes, maxRes);
+  }
+
   public static ResourceInformation newInstance(String name, long value) {
     return ResourceInformation
         .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fdbe2d4..3a9662b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -551,6 +551,10 @@ message QueueConfigurationsProto {
     optional float maxCapacity = 3;
     optional float absoluteMaxCapacity = 4;
     optional float maxAMPercentage = 5;
+    optional ResourceProto effectiveMinCapacity = 6;
+    optional ResourceProto effectiveMaxCapacity = 7;
+    optional ResourceProto configuredMinCapacity = 8;
+    optional ResourceProto configuredMaxCapacity = 9;
 }
 
 message QueueConfigurationsMapProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index f308bce..80ef4b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -19,16 +19,22 @@
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
 import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 
 import com.google.protobuf.TextFormat;
 
 public class QueueConfigurationsPBImpl extends QueueConfigurations {
 
-  QueueConfigurationsProto proto =
-      QueueConfigurationsProto.getDefaultInstance();
+  QueueConfigurationsProto proto = QueueConfigurationsProto
+      .getDefaultInstance();
   QueueConfigurationsProto.Builder builder = null;
+  Resource configuredMinResource = null;
+  Resource configuredMaxResource = null;
+  Resource effMinResource = null;
+  Resource effMaxResource = null;
   boolean viaProto = false;
 
   public QueueConfigurationsPBImpl() {
@@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
   }
 
   public QueueConfigurationsProto getProto() {
+    mergeLocalToProto();
     proto = viaProto ? proto : builder.build();
     viaProto = true;
     return proto;
   }
 
+  private void mergeLocalToProto() {
+    if (viaProto) {
+      maybeInitBuilder();
+    }
+    mergeLocalToBuilder();
+    proto = builder.build();
+    viaProto = true;
+  }
+
+  private void mergeLocalToBuilder() {
+    if (this.effMinResource != null) {
+      builder
+          .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource));
+    }
+    if (this.effMaxResource != null) {
+      builder
+          .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
+    }
+    if (this.configuredMinResource != null) {
+      builder.setEffectiveMinCapacity(
+          convertToProtoFormat(this.configuredMinResource));
+    }
+    if (this.configuredMaxResource != null) {
+      builder.setEffectiveMaxCapacity(
+          convertToProtoFormat(this.configuredMaxResource));
+    }
+  }
+
   @Override
   public float getCapacity() {
     QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
@@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
     builder.setMaxAMPercentage(maxAMPercentage);
   }
 
+  @Override
+  public Resource getEffectiveMinCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.effMinResource != null) {
+      return this.effMinResource;
+    }
+    if (!p.hasEffectiveMinCapacity()) {
+      return null;
+    }
+    this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity());
+    return this.effMinResource;
+  }
+
+  @Override
+  public void setEffectiveMinCapacity(Resource capacity) {
+    maybeInitBuilder();
+    if (capacity == null) {
+      builder.clearEffectiveMinCapacity();
+    }
+    this.effMinResource = capacity;
+  }
+
+  @Override
+  public Resource getEffectiveMaxCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.effMaxResource != null) {
+      return this.effMaxResource;
+    }
+    if (!p.hasEffectiveMaxCapacity()) {
+      return null;
+    }
+    this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity());
+    return this.effMaxResource;
+  }
+
+  @Override
+  public void setEffectiveMaxCapacity(Resource capacity) {
+    maybeInitBuilder();
+    if (capacity == null) {
+      builder.clearEffectiveMaxCapacity();
+    }
+    this.effMaxResource = capacity;
+  }
+
+  private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+    return new ResourcePBImpl(p);
+  }
+
+  private ResourceProto convertToProtoFormat(Resource t) {
+    return ProtoUtils.convertToProtoFormat(t);
+  }
+
   private void maybeInitBuilder() {
     if (viaProto || builder == null) {
       builder = QueueConfigurationsProto.newBuilder(proto);
@@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
     return false;
   }
 
+  @Override
+  public Resource getConfiguredMinCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.configuredMinResource != null) {
+      return this.configuredMinResource;
+    }
+    if (!p.hasConfiguredMinCapacity()) {
+      return null;
+    }
+    this.configuredMinResource = convertFromProtoFormat(
+        p.getConfiguredMinCapacity());
+    return this.configuredMinResource;
+  }
+
+  @Override
+  public void setConfiguredMinCapacity(Resource configuredMinResource) {
+    maybeInitBuilder();
+    if (configuredMinResource == null) {
+      builder.clearConfiguredMinCapacity();
+    }
+    this.configuredMinResource = configuredMinResource;
+  }
+
+  @Override
+  public Resource getConfiguredMaxCapacity() {
+    QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+    if (this.configuredMaxResource != null) {
+      return this.configuredMaxResource;
+    }
+    if (!p.hasConfiguredMaxCapacity()) {
+      return null;
+    }
+    this.configuredMaxResource = convertFromProtoFormat(
+        p.getConfiguredMaxCapacity());
+    return this.configuredMaxResource;
+  }
+
+  @Override
+  public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+    maybeInitBuilder();
+    if (configuredMaxResource == null) {
+      builder.clearConfiguredMaxCapacity();
+    }
+    this.configuredMaxResource = configuredMaxResource;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 16e4527..cc4acf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
 
   @Override
   public Resource normalizeDown(Resource r, Resource stepFactor) {
-    return Resources.createResource(
-        roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
-        roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+    Resource ret = Resource.newInstance(r);
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+    for (int i = 0; i < maxLength; i++) {
+      ResourceInformation rResourceInformation = r.getResourceInformation(i);
+      ResourceInformation stepFactorResourceInformation = stepFactor
+          .getResourceInformation(i);
+      ResourceInformation tmp = ret.getResourceInformation(i);
+
+      long rValue = rResourceInformation.getValue();
+      long stepFactorValue = UnitsConversionUtil.convert(
+          stepFactorResourceInformation.getUnits(),
+          rResourceInformation.getUnits(),
+          stepFactorResourceInformation.getValue());
+
+      long value = rValue;
+      if (stepFactorValue != 0) {
+        value = roundDown(rValue, stepFactorValue);
+      }
+      tmp.setValue(value);
+    }
+    return ret;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
index 2e653fc..08b4d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage {
   public void setEffectiveMaxResource(String label, Resource res) {
     _set(label, ResourceType.EFF_MAX_RESOURCE, res);
   }
-
-  /*
-   * Effective Minimum Resource
-   */
-  public Resource getEffectiveMinResourceUp() {
-    return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
-  }
-
-  public Resource getEffectiveMinResourceUp(String label) {
-    return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
-  }
-
-  public void setEffectiveMinResourceUp(String label, Resource res) {
-    _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
-  }
-
-  public void setEffectiveMinResourceUp(Resource res) {
-    _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
-  }
-
-  /*
-   * Effective Maximum Resource
-   */
-  public Resource getEffectiveMaxResourceUp() {
-    return getEffectiveMaxResourceUp(NL);
-  }
-
-  public Resource getEffectiveMaxResourceUp(String label) {
-    return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
-  }
-
-  public void setEffectiveMaxResourceUp(Resource res) {
-    setEffectiveMaxResourceUp(NL, res);
-  }
-
-  public void setEffectiveMaxResourceUp(String label, Resource res) {
-    _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index dddac4d..9caf589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue {
       Resource maxResource = conf.getMaximumResourceRequirement(label,
           queuePath, resourceTypes);
 
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("capacityConfigType is '" + capacityConfigType
+            + "' for queue '" + getQueueName());
+      }
       if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
         this.capacityConfigType = (!minResource.equals(Resources.none())
             && queueCapacities.getAbsoluteCapacity(label) == 0f)
@@ -481,12 +485,6 @@ public abstract class AbstractCSQueue implements CSQueue {
   }
 
   @Override
-  public Resource getEffectiveCapacityUp(String label) {
-    return Resources
-        .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
-  }
-
-  @Override
   public Resource getEffectiveCapacityDown(String label, Resource factor) {
     return Resources.normalizeDown(resourceCalculator,
         getQueueResourceQuotas().getEffectiveMinResource(label),
@@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue {
       queueConfiguration.setMaxCapacity(maxCapacity);
       queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
       queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+      queueConfiguration.setConfiguredMinCapacity(
+          queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+      queueConfiguration.setConfiguredMaxCapacity(
+          queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+      queueConfiguration.setEffectiveMinCapacity(
+          queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+      queueConfiguration.setEffectiveMaxCapacity(
+          queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
       queueConfigurations.put(nodeLabel, queueConfiguration);
     }
     return queueConfigurations;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 2e29a71..624fcc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -25,7 +25,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Stable;
 import org.apache.hadoop.security.AccessControlException;
@@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * <code>CSQueue</code> represents a node in the tree of 
  * hierarchical queues in the {@link CapacityScheduler}.
@@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    * @return effective queue capacity
    */
   Resource getEffectiveCapacity(String label);
-  Resource getEffectiveCapacityUp(String label);
   Resource getEffectiveCapacityDown(String label, Resource factor);
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1460121..41ec4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
           1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
       effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
 
-      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
 
       Resource userAMLimit = Resources.multiplyAndNormalizeUp(
           resourceCalculator, queuePartitionResource,
@@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
        * non-labeled), * with per-partition am-resource-percent to get the max am
        * resource limit for this queue and partition.
        */
-      Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+      Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
 
       Resource queueCurrentLimit = Resources.none();
       // For non-labeled partition, we need to consider the current queue

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index c770fac..5ab1494 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue {
       queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
       queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
       queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+      queueCapacities.setAbsoluteCapacity(label, 1.0f);
     }
 
-    // Total configured min resources of direct children of queue
+    // Total configured min resources of direct children of this given parent
+    // queue.
     Resource configuredMinResources = Resource.newInstance(0L, 0);
     for (CSQueue childQueue : getChildQueues()) {
       Resources.addTo(configuredMinResources,
@@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue {
         // present could also be taken from effective max resource of parent.
         Resource parentMaxRes = queueResourceQuotas
             .getConfiguredMaxResource(label);
-        if (parentMaxRes.equals(Resources.none())) {
+        if (parent != null && parentMaxRes.equals(Resources.none())) {
           parentMaxRes = parent.getQueueResourceQuotas()
               .getEffectiveMaxResource(label);
         }
@@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue {
             parentMaxRes);
         childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
             Resources.clone(effMaxResource));
+
+        // In cases where we still need to update some units based on
+        // percentage, we have to calculate percentage and update.
+        deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc,
+            childQueue);
       } else {
         childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
             Resources.multiply(resourceByLabel,
@@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue {
         childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
             Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
                 .getAbsoluteMaximumCapacity(label)));
-
-        childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
-            Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
-                childQueue.getQueueCapacities().getAbsoluteCapacity(label),
-                minimumAllocation));
-        childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
-            Resources.multiplyAndNormalizeUp(rc,
-                resourceByLabel, childQueue.getQueueCapacities()
-                    .getAbsoluteMaximumCapacity(label),
-                    minimumAllocation));
       }
 
       if (LOG.isDebugEnabled()) {
@@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue {
     }
   }
 
+  private void deriveCapacityFromAbsoluteConfigurations(String label,
+      Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
+
+    /*
+     * In case when queues are configured with absolute resources, it is better
+     * to update capacity/max-capacity etc w.r.t absolute resource as well. In
+     * case of computation, these values wont be used any more. However for
+     * metrics and UI, its better these values are pre-computed here itself.
+     */
+
+    // 1. Update capacity as a float based on parent's minResource
+    childQueue.getQueueCapacities().setCapacity(label,
+        rc.divide(clusterResource,
+            childQueue.getQueueResourceQuotas().getEffectiveMinResource(label),
+            getQueueResourceQuotas().getEffectiveMinResource(label)));
+
+    // 2. Update max-capacity as a float based on parent's maxResource
+    childQueue.getQueueCapacities().setMaximumCapacity(label,
+        rc.divide(clusterResource,
+            childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label),
+            getQueueResourceQuotas().getEffectiveMaxResource(label)));
+
+    // 3. Update absolute capacity as a float based on parent's minResource and
+    // cluster resource.
+    childQueue.getQueueCapacities().setAbsoluteCapacity(label,
+        (float) childQueue.getQueueCapacities().getCapacity()
+            / getQueueCapacities().getAbsoluteCapacity(label));
+
+    // 4. Update absolute max-capacity as a float based on parent's maxResource
+    // and cluster resource.
+    childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
+        (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
+            / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+  }
+
   @Override
   public List<CSQueue> getChildQueues() {
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index efc20e9..7287c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager {
      * * If we're running over capacity, then its (usedResources + required)
      * (which extra resources we are allocating)
      */
-    Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
+    Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition);
 
     /*
      * Assume we have required resource equals to minimumAllocation, this can

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 34594cf..a4e0096 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -909,6 +910,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
       StringBuilder diagnosticMessage) {
     LeafQueue queue = getCSLeafQueue();
     QueueCapacities queueCapacities = queue.getQueueCapacities();
+    QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
     diagnosticMessage.append(" Details : AM Partition = ");
     diagnosticMessage.append(appAMNodePartitionName.isEmpty()
         ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
@@ -930,6 +932,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
         queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
             * 100);
     diagnosticMessage.append(" % ; ");
+    diagnosticMessage.append("Queue's capacity (absolute resource) = ");
+    diagnosticMessage.append(
+        queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
+    diagnosticMessage.append("Queue's used capacity (absolute resource) = ");
+    diagnosticMessage
+        .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
+    diagnosticMessage.append("Queue's max capacity (absolute resource) = ");
+    diagnosticMessage.append(
+        queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName));
+    diagnosticMessage.append(" ; ");
   }
 
   /**
@@ -993,13 +1007,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
       ResourceCalculator calc =
           rmContext.getScheduler().getResourceCalculator();
       if (!calc.isInvalidDivisor(totalPartitionRes)) {
-        float queueAbsMaxCapPerPartition =
-            ((AbstractCSQueue) getQueue()).getQueueCapacities()
-                .getAbsoluteCapacity(getAppAMNodePartitionName());
+        Resource effCap = ((AbstractCSQueue) getQueue())
+            .getEffectiveCapacity(getAppAMNodePartitionName());
         float queueUsagePerc = calc.divide(totalPartitionRes,
-            report.getUsedResources(),
-            Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
-            * 100;
+            report.getUsedResources(), effCap) * 100;
         report.setQueueUsagePercentage(queueUsagePerc);
       }
       return report;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 74b4e79..3f68e07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView {
           ? new ResourceInfo(Resources.none())
           : resourceUsages.getAmUsed();
       ri.
-          __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
-          __("Configured Capacity:", percent(capacities.getCapacity() / 100)).
-          __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)).
-          __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
-          __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
-          __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
-          __("Used Resources:", resourceUsages.getUsed().toString()).
-          __("Configured Max Application Master Limit:", StringUtils.format("%.1f",
-          capacities.getMaxAMLimitPercentage())).
-          __("Max Application Master Resources:",
-          resourceUsages.getAMLimit().toString()).
-          __("Used Application Master Resources:",
-          amUsed.toString()).
-          __("Max Application Master Resources Per User:",
-          userAMResourceLimit.toString());
+          __("Used Capacity:",
+              appendPercent(resourceUsages.getUsed().toString(),
+                  capacities.getUsedCapacity() / 100))
+          .__("Configured Capacity:",
+              capacities.getConfiguredMinResource().toString())
+          .__("Configured Max Capacity:",
+              capacities.getConfiguredMaxResource().getResource()
+                  .equals(Resources.none())
+                      ? "unlimited"
+                      : capacities.getConfiguredMaxResource().toString())
+          .__("Effective Capacity:",
+              appendPercent(capacities.getEffectiveMinResource().toString(),
+                  capacities.getCapacity() / 100))
+          .__("Effective Max Capacity:",
+              appendPercent(capacities.getEffectiveMaxResource().toString(),
+                  capacities.getMaxCapacity() / 100))
+          .__("Absolute Used Capacity:",
+              percent(capacities.getAbsoluteUsedCapacity() / 100))
+          .__("Absolute Configured Capacity:",
+              percent(capacities.getAbsoluteCapacity() / 100))
+          .__("Absolute Configured Max Capacity:",
+              percent(capacities.getAbsoluteMaxCapacity() / 100))
+          .__("Used Resources:", resourceUsages.getUsed().toString())
+          .__("Configured Max Application Master Limit:",
+              StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage()))
+          .__("Max Application Master Resources:",
+              resourceUsages.getAMLimit().toString())
+          .__("Used Application Master Resources:", amUsed.toString())
+          .__("Max Application Master Resources Per User:",
+              userAMResourceLimit.toString());
     }
 
     private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView {
     return QueuesBlock.class;
   }
 
+  static String appendPercent(String message, float f) {
+    return message + " (" + StringUtils.formatPercent(f, 1) + ")";
+  }
+
   static String percent(float f) {
     return StringUtils.formatPercent(f, 1);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 32e4ac5..ca42041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
       max = 1f;
     this.maxCapacity = max * 100;
 
-    capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
+    capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
+        parent.getQueueResourceQuotas(), false);
     queues = getQueues(parent);
     health = new CapacitySchedulerHealthInfo(cs);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 7dcdf58..343c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
   }
 
   @Override
-  protected void populateQueueCapacities(QueueCapacities qCapacities) {
-    capacities = new QueueCapacitiesInfo(qCapacities);
+  protected void populateQueueCapacities(QueueCapacities qCapacities,
+      QueueResourceQuotas qResQuotas) {
+    capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas);
   }
 
   public int getNumActiveApplications() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 86b2fea..d4de9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.XmlTransient;
 
 import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo {
       Collections.sort(nodeLabels);
     }
     QueueCapacities qCapacities = q.getQueueCapacities();
-    populateQueueCapacities(qCapacities);
+    QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas();
+    populateQueueCapacities(qCapacities, qResQuotas);
 
     ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
     populateQueueResourceUsage(queueResourceUsage);
@@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo {
     resources = new ResourcesInfo(queueResourceUsage, false);
   }
 
-  protected void populateQueueCapacities(QueueCapacities qCapacities) {
-    capacities = new QueueCapacitiesInfo(qCapacities, false);
+  protected void populateQueueCapacities(QueueCapacities qCapacities,
+      QueueResourceQuotas qResQuotas) {
+    capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas,
+        false);
   }
 
   public float getCapacity() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
index 5e298f9..2a15502 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
@@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
 /**
  * This class represents queue capacities for a given partition
  */
@@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo {
   private float absoluteUsedCapacity;
   private float absoluteMaxCapacity  = 100;
   private float maxAMLimitPercentage;
+  private ResourceInfo configuredMinResource;
+  private ResourceInfo configuredMaxResource;
+  private ResourceInfo effectiveMinResource;
+  private ResourceInfo effectiveMaxResource;
 
   public PartitionQueueCapacitiesInfo() {
   }
 
   public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
       float usedCapacity, float maxCapacity, float absCapacity,
-      float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) {
+      float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
+      Resource confMinRes, Resource confMaxRes, Resource effMinRes,
+      Resource effMaxRes) {
     super();
     this.partitionName = partitionName;
     this.capacity = capacity;
@@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo {
     this.absoluteUsedCapacity = absUsedCapacity;
     this.absoluteMaxCapacity = absMaxCapacity;
     this.maxAMLimitPercentage = maxAMLimitPercentage;
+    this.configuredMinResource = new ResourceInfo(confMinRes);
+    this.configuredMaxResource = new ResourceInfo(confMaxRes);
+    this.effectiveMinResource = new ResourceInfo(effMinRes);
+    this.effectiveMaxResource = new ResourceInfo(effMaxRes);
   }
 
   public float getCapacity() {
@@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo {
   public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
     this.maxAMLimitPercentage = maxAMLimitPercentage;
   }
+
+  public ResourceInfo getConfiguredMinResource() {
+    return configuredMinResource;
+  }
+
+  public ResourceInfo getConfiguredMaxResource() {
+    if (configuredMaxResource.getResource().equals(Resources.none())) {
+      return null;
+    }
+    return configuredMaxResource;
+  }
+
+  public ResourceInfo getEffectiveMinResource() {
+    return effectiveMinResource;
+  }
+
+  public ResourceInfo getEffectiveMaxResource() {
+    return effectiveMaxResource;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
index 9a3e439..35c80d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 
 /**
@@ -39,6 +40,7 @@ public class QueueCapacitiesInfo {
   }
 
   public QueueCapacitiesInfo(QueueCapacities capacities,
+      QueueResourceQuotas resourceQuotas,
       boolean considerAMUsage) {
     if (capacities == null) {
       return;
@@ -68,12 +70,17 @@ public class QueueCapacitiesInfo {
       queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
           partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
           absUsedCapacity, absMaxCapacity,
-          considerAMUsage ? maxAMLimitPercentage : 0f));
+          considerAMUsage ? maxAMLimitPercentage : 0f,
+          resourceQuotas.getConfiguredMinResource(partitionName),
+          resourceQuotas.getConfiguredMaxResource(partitionName),
+          resourceQuotas.getEffectiveMinResource(partitionName),
+          resourceQuotas.getEffectiveMaxResource(partitionName)));
     }
   }
 
-  public QueueCapacitiesInfo(QueueCapacities capacities) {
-    this(capacities, true);
+  public QueueCapacitiesInfo(QueueCapacities capacities,
+      QueueResourceQuotas resourceQuotas) {
+    this(capacities, resourceQuotas, true);
   }
 
   public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index b386c18..38768e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     Map<String, ResourceInformation> riMap = new HashMap<>();
 
     // Initialize mandatory resources
-    riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
-    riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
-    riMap.put(RESOURCE_1, ResourceInformation
-        .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
-            Integer.MAX_VALUE));
+    ResourceInformation memory = ResourceInformation.newInstance(
+        ResourceInformation.MEMORY_MB.getName(),
+        ResourceInformation.MEMORY_MB.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+    ResourceInformation vcores = ResourceInformation.newInstance(
+        ResourceInformation.VCORES.getName(),
+        ResourceInformation.VCORES.getUnits(),
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+        YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+    riMap.put(ResourceInformation.MEMORY_URI, memory);
+    riMap.put(ResourceInformation.VCORES_URI, vcores);
+    riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+        ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
 
     ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
 
@@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
     MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
 
-    Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+    Assert.assertEquals(Resource.newInstance(1 * GB, 1),
         leafQueue.getUsedResources());
 
     RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
     // Do node heartbeats 1 time and check container allocated.
     cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
 
-    // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+    // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
     Assert.assertEquals(
-        TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+        TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
         leafQueue.getUsedResources());
 
     // Acquire container

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d52627a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 96f8970..30bff78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -1355,7 +1356,7 @@ public class TestLeafQueue {
     // TODO, fix headroom in the future patch
     assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
       // User limit = 2G, 2 in use
-    assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+    assertEquals(1*GB, app_1.getHeadroom().getMemorySize());
       // the application is not yet active
 
     // Again one to user_0 since he hasn't exceeded user limit yet
@@ -1366,8 +1367,8 @@ public class TestLeafQueue {
     assertEquals(3*GB, a.getUsedResources().getMemorySize());
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-    assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
-    assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+    assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+    assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
     
     // Submit requests for app_1 and set max-cap
     a.setMaxCapacity(.1f);
@@ -3874,7 +3875,7 @@ public class TestLeafQueue {
 
     // Queue "test" consumes 100% of the cluster, so its capacity and absolute
     // capacity are both 1.0f.
-    Queue queue = createQueue("test", null, 1.0f, 1.0f);
+    Queue queue = createQueue("test", null, 1.0f, 1.0f, res);
     final String user = "user1";
     FiCaSchedulerApp app =
         new FiCaSchedulerApp(appAttId, user, queue,
@@ -3891,7 +3892,8 @@ public class TestLeafQueue {
 
     // Queue "test2" is a child of root and its capacity is 50% of root. As a
     // child of root, its absolute capaicty is also 50%.
-    queue = createQueue("test2", null, 0.5f, 0.5f);
+    queue = createQueue("test2", null, 0.5f, 0.5f,
+        Resources.divideAndCeil(dominantResourceCalculator, res, 2));
     app = new FiCaSchedulerApp(appAttId, user, queue,
         queue.getAbstractUsersManager(), rmContext);
     app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3903,7 +3905,8 @@ public class TestLeafQueue {
 
     // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
     // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
-    AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+    AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f,
+        Resources.divideAndCeil(dominantResourceCalculator, res, 4));
     app = new FiCaSchedulerApp(appAttId, user, qChild,
         qChild.getAbstractUsersManager(), rmContext);
     app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3922,7 +3925,7 @@ public class TestLeafQueue {
   }
 
   private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
-      float absCap) {
+      float absCap, Resource res) {
     CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
     QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
         null, QueueState.RUNNING, null, "", null, false);
@@ -3934,6 +3937,13 @@ public class TestLeafQueue {
     QueueCapacities qCaps = mock(QueueCapacities.class);
     when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
     when(queue.getQueueCapacities()).thenReturn(qCaps);
+    QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class);
+    when(qQuotas.getConfiguredMinResource(any())).thenReturn(res);
+    when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res);
+    when(qQuotas.getEffectiveMinResource(any())).thenReturn(res);
+    when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res);
+    when(queue.getQueueResourceQuotas()).thenReturn(qQuotas);
+    when(queue.getEffectiveCapacity(any())).thenReturn(res);
     return queue;
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[26/50] hadoop git commit: HADOOP-15059. Undoing the switch of Credentials to PB format as default - done via HADOOP-12563 for supporting 2.x to 3.x upgrades.

Posted by xy...@apache.org.
HADOOP-15059. Undoing the switch of Credentials to PB format as default - done via HADOOP-12563 for supporting 2.x to 3.x upgrades.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1963833
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1963833
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1963833

Branch: refs/heads/HDFS-7240
Commit: f19638333b11da6dcab9a964e73a49947b8390fd
Parents: ce04340
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Fri Dec 8 08:00:21 2017 -0800
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Fri Dec 8 08:00:21 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/security/Credentials.java | 112 ++++++++++++-------
 .../hadoop/security/token/DtFileOperations.java |  10 +-
 .../hadoop/security/token/TestDtUtilShell.java  |  15 +--
 .../hdfs/tools/DelegationTokenFetcher.java      |   3 +-
 4 files changed, 86 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1963833/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 4d58981..3e51249 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -27,7 +27,6 @@ import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
@@ -60,6 +59,28 @@ import org.slf4j.LoggerFactory;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class Credentials implements Writable {
+  public enum SerializedFormat {
+    WRITABLE((byte) 0x00),
+    PROTOBUF((byte) 0x01);
+
+    // Caching to avoid reconstructing the array each time.
+    private static final SerializedFormat[] FORMATS = values();
+
+    final byte value;
+
+    SerializedFormat(byte val) {
+      this.value = val;
+    }
+
+    public static SerializedFormat valueOf(int val) {
+      try {
+        return FORMATS[val];
+      } catch (ArrayIndexOutOfBoundsException e) {
+        throw new IllegalArgumentException("Unknown credential format: " + val);
+      }
+    }
+  }
+
   private static final Logger LOG = LoggerFactory.getLogger(Credentials.class);
 
   private  Map<Text, byte[]> secretKeysMap = new HashMap<Text, byte[]>();
@@ -224,63 +245,74 @@ public class Credentials implements Writable {
     if (!Arrays.equals(magic, TOKEN_STORAGE_MAGIC)) {
       throw new IOException("Bad header found in token storage.");
     }
-    byte version = in.readByte();
-    if (version != TOKEN_STORAGE_VERSION &&
-        version != OLD_TOKEN_STORAGE_VERSION) {
-      throw new IOException("Unknown version " + version +
-                            " in token storage.");
+    SerializedFormat format;
+    try {
+      format = SerializedFormat.valueOf(in.readByte());
+    } catch (IllegalArgumentException e) {
+      throw new IOException(e);
     }
-    if (version == OLD_TOKEN_STORAGE_VERSION) {
+    switch (format) {
+    case WRITABLE:
       readFields(in);
-    } else if (version == TOKEN_STORAGE_VERSION) {
+      break;
+    case PROTOBUF:
       readProto(in);
+      break;
+    default:
+      throw new IOException("Unsupported format " + format);
     }
   }
 
   private static final byte[] TOKEN_STORAGE_MAGIC =
       "HDTS".getBytes(StandardCharsets.UTF_8);
-  private static final byte TOKEN_STORAGE_VERSION = 1;
-
-  /**
-   *  For backward compatibility.
-   */
-  private static final byte OLD_TOKEN_STORAGE_VERSION = 0;
-
 
   public void writeTokenStorageToStream(DataOutputStream os)
       throws IOException {
-    os.write(TOKEN_STORAGE_MAGIC);
-    os.write(TOKEN_STORAGE_VERSION);
-    writeProto(os);
+    // by default store in the oldest supported format for compatibility
+    writeTokenStorageToStream(os, SerializedFormat.WRITABLE);
   }
 
-  public void writeTokenStorageFile(Path filename,
-                                    Configuration conf) throws IOException {
-    FSDataOutputStream os = filename.getFileSystem(conf).create(filename);
-    writeTokenStorageToStream(os);
-    os.close();
+  public void writeTokenStorageToStream(DataOutputStream os,
+      SerializedFormat format) throws IOException {
+    switch (format) {
+    case WRITABLE:
+      writeWritableOutputStream(os);
+      break;
+    case PROTOBUF:
+      writeProtobufOutputStream(os);
+      break;
+    default:
+      throw new IllegalArgumentException("Unsupported serialized format: "
+          + format);
+    }
   }
 
-  /**
-   *  For backward compatibility.
-   */
-  public void writeLegacyTokenStorageLocalFile(File f) throws IOException {
-    writeLegacyOutputStream(new DataOutputStream(new FileOutputStream(f)));
+  private void writeWritableOutputStream(DataOutputStream os)
+      throws IOException {
+    os.write(TOKEN_STORAGE_MAGIC);
+    os.write(SerializedFormat.WRITABLE.value);
+    write(os);
   }
 
-  /**
-   *  For backward compatibility.
-   */
-  public void writeLegacyTokenStorageFile(Path filename, Configuration conf)
+  private void writeProtobufOutputStream(DataOutputStream os)
       throws IOException {
-    writeLegacyOutputStream(filename.getFileSystem(conf).create(filename));
+    os.write(TOKEN_STORAGE_MAGIC);
+    os.write(SerializedFormat.PROTOBUF.value);
+    writeProto(os);
   }
 
-  private void writeLegacyOutputStream(DataOutputStream os) throws IOException {
-    os.write(TOKEN_STORAGE_MAGIC);
-    os.write(OLD_TOKEN_STORAGE_VERSION);
-    write(os);
-    os.close();
+  public void writeTokenStorageFile(Path filename,
+                                    Configuration conf) throws IOException {
+    // by default store in the oldest supported format for compatibility
+    writeTokenStorageFile(filename, conf, SerializedFormat.WRITABLE);
+  }
+
+  public void writeTokenStorageFile(Path filename, Configuration conf,
+      SerializedFormat format) throws IOException {
+    try (FSDataOutputStream os =
+             filename.getFileSystem(conf).create(filename)) {
+      writeTokenStorageToStream(os, format);
+    }
   }
 
   /**
@@ -312,7 +344,7 @@ public class Credentials implements Writable {
    * @param out
    * @throws IOException
    */
-  public void writeProto(DataOutput out) throws IOException {
+  void writeProto(DataOutput out) throws IOException {
     CredentialsProto.Builder storage = CredentialsProto.newBuilder();
     for (Map.Entry<Text, Token<? extends TokenIdentifier>> e :
                                                          tokenMap.entrySet()) {
@@ -337,7 +369,7 @@ public class Credentials implements Writable {
    * Populates keys/values from proto buffer storage.
    * @param in - stream ready to read a serialized proto buffer message
    */
-  public void readProto(DataInput in) throws IOException {
+  void readProto(DataInput in) throws IOException {
     CredentialsProto storage = CredentialsProto.parseDelimitedFrom((DataInputStream)in);
     for (CredentialsKVProto kv : storage.getTokensList()) {
       addToken(new Text(kv.getAliasBytes().toByteArray()),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1963833/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index d128cc9..d36ad9b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -102,11 +102,13 @@ public final class DtFileOperations {
   public static void doFormattedWrite(
       File f, String format, Credentials creds, Configuration conf)
       throws IOException {
-    if (format == null || format.equals(FORMAT_PB)) {
-      creds.writeTokenStorageFile(fileToPath(f), conf);
-    } else { // if (format != null && format.equals(FORMAT_JAVA)) {
-      creds.writeLegacyTokenStorageLocalFile(f);
+    // default to oldest supported format for compatibility
+    Credentials.SerializedFormat credsFormat =
+        Credentials.SerializedFormat.WRITABLE;
+    if (format.equals(FORMAT_PB)) {
+      credsFormat = Credentials.SerializedFormat.PROTOBUF;
     }
+    creds.writeTokenStorageFile(fileToPath(f), conf, credsFormat);
   }
 
   /** Print out a Credentials file from the local filesystem.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1963833/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
index 1c25912..53b0d3d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.security.token;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.FileInputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
 
@@ -29,14 +28,12 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.security.token.DtFetcher;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -54,7 +51,6 @@ public class TestDtUtilShell {
   private static Configuration defaultConf = new Configuration();
   private static FileSystem localFs = null;
   private final String alias = "proxy_ip:1234";
-  private final String renewer = "yarn";
   private final String getUrl = SERVICE_GET.toString() + "://localhost:9000/";
   private final String getUrl2 = "http://localhost:9000/";
   public static Text SERVICE_GET = new Text("testTokenServiceGet");
@@ -111,11 +107,12 @@ public class TestDtUtilShell {
     Token<? extends TokenIdentifier> tok = (Token<? extends TokenIdentifier>)
         new Token(IDENTIFIER, PASSWORD, KIND, service);
     creds.addToken(tok.getService(), tok);
+    Credentials.SerializedFormat format =
+        Credentials.SerializedFormat.PROTOBUF;
     if (legacy) {
-      creds.writeLegacyTokenStorageLocalFile(new File(tokenPath.toString()));
-    } else {
-      creds.writeTokenStorageFile(tokenPath, defaultConf);
+      format = Credentials.SerializedFormat.WRITABLE;
     }
+    creds.writeTokenStorageFile(tokenPath, defaultConf, format);
   }
 
   @Test
@@ -284,6 +281,6 @@ public class TestDtUtilShell {
     DataInputStream in = new DataInputStream(
         new FileInputStream(tokenFilenameGet));
     spyCreds.readTokenStorageStream(in);
-    Mockito.verify(spyCreds).readProto(in);
+    Mockito.verify(spyCreds, Mockito.never()).readFields(in);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1963833/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index cf77804..c6ea91c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -182,7 +182,8 @@ public class DelegationTokenFetcher {
       Credentials cred = new Credentials();
       cred.addToken(token.getService(), token);
       // dtutil is replacing this tool; preserve legacy functionality
-      cred.writeLegacyTokenStorageFile(tokenFile, conf);
+      cred.writeTokenStorageFile(tokenFile, conf,
+          Credentials.SerializedFormat.WRITABLE);
 
       if (LOG.isDebugEnabled()) {
         LOG.debug("Fetched token " + fs.getUri() + " for " +


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[22/50] hadoop git commit: YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 0544387..4985a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -121,6 +123,15 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
       // For queue with same used ratio / priority, queue with higher configured
       // capacity goes first
       if (0 == rc) {
+        Resource minEffRes1 = q1.getQueueResourceQuotas()
+            .getConfiguredMinResource(p);
+        Resource minEffRes2 = q2.getQueueResourceQuotas()
+            .getConfiguredMinResource(p);
+        if (!minEffRes1.equals(Resources.none())
+            && !minEffRes2.equals(Resources.none())) {
+          return minEffRes2.compareTo(minEffRes1);
+        }
+
         float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
         float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
         return Float.compare(abs2, abs1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 22705cc..86b2fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -62,6 +62,8 @@ public class CapacitySchedulerQueueInfo {
   protected long pendingContainers;
   protected QueueCapacitiesInfo capacities;
   protected ResourcesInfo resources;
+  protected ResourceInfo minEffectiveCapacity;
+  protected ResourceInfo maxEffectiveCapacity;
 
   CapacitySchedulerQueueInfo() {
   };
@@ -105,6 +107,11 @@ public class CapacitySchedulerQueueInfo {
 
     ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
     populateQueueResourceUsage(queueResourceUsage);
+
+    minEffectiveCapacity = new ResourceInfo(
+        q.getQueueResourceQuotas().getEffectiveMinResource());
+    maxEffectiveCapacity = new ResourceInfo(
+        q.getQueueResourceQuotas().getEffectiveMaxResource());
   }
 
   protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -200,4 +207,12 @@ public class CapacitySchedulerQueueInfo {
   public ResourcesInfo getResources() {
     return resources;
   }
+
+  public ResourceInfo getMinEffectiveCapacity(){
+    return minEffectiveCapacity;
+  }
+
+  public ResourceInfo getMaxEffectiveCapacity(){
+    return maxEffectiveCapacity;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index b5c20f4..909d023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
 import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
 import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
 import org.apache.hadoop.yarn.server.api.records.MasterKey;
 import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -142,6 +143,13 @@ public class MockNM {
     return this.registeringCollectors;
   }
 
+  public void unRegisterNode() throws Exception {
+    UnRegisterNodeManagerRequest request = Records
+        .newRecord(UnRegisterNodeManagerRequest.class);
+    request.setNodeId(nodeId);
+    resourceTracker.unRegisterNodeManager(request);
+  }
+
   public RegisterNodeManagerResponse registerNode() throws Exception {
     return registerNode(null, null);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 711f008..19ca6d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -823,6 +823,12 @@ public class MockRM extends ResourceManager {
     return rmApp;
   }
 
+  public MockNM unRegisterNode(MockNM nm) throws Exception {
+    nm.unRegisterNode();
+    drainEventsImplicitly();
+    return nm;
+  }
+
   public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
     MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
     nm.registerNode();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 0bc5cb5..1dda8a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
@@ -644,9 +645,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
 
     QueueCapacities qc = new QueueCapacities(0 == myLevel);
     ResourceUsage ru = new ResourceUsage();
+    QueueResourceQuotas qr  = new QueueResourceQuotas();
 
     when(queue.getQueueCapacities()).thenReturn(qc);
     when(queue.getQueueResourceUsage()).thenReturn(ru);
+    when(queue.getQueueResourceQuotas()).thenReturn(qr);
 
     LOG.debug("Setup queue, name=" + queue.getQueueName() + " path="
         + queue.getQueuePath());
@@ -679,7 +682,17 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
       qc.setAbsoluteMaximumCapacity(partitionName, absMax);
       qc.setAbsoluteUsedCapacity(partitionName, absUsed);
       qc.setUsedCapacity(partitionName, used);
+      qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim()));
+      qr.setEffectiveMinResource(parseResourceFromString(values[0].trim()));
+      qr.setEffectiveMaxResource(partitionName,
+          parseResourceFromString(values[1].trim()));
+      qr.setEffectiveMinResource(partitionName,
+          parseResourceFromString(values[0].trim()));
       when(queue.getUsedCapacity()).thenReturn(used);
+      when(queue.getEffectiveCapacity(partitionName))
+          .thenReturn(parseResourceFromString(values[0].trim()));
+      when(queue.getEffectiveMaxCapacity(partitionName))
+          .thenReturn(parseResourceFromString(values[1].trim()));
       ru.setPending(partitionName, pending);
       // Setup reserved resource if it contained by input config
       Resource reserved = Resources.none();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index f0ca466..22a241f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
 import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitorManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -437,8 +438,8 @@ public class TestProportionalCapacityPreemptionPolicy {
     policy.editSchedule();
     // queueF(appD) wants resources, Verify that resources come from queueE(appC)
     // because it's a sibling and queueB(appA) because queueA is over capacity.
-    verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA)));
-    verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC)));
+    verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA)));
+    verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC)));
 
     // Need to call setup() again to reset mDisp
     setup();
@@ -1204,6 +1205,17 @@ public class TestProportionalCapacityPreemptionPolicy {
     when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
     boolean preemptionDisabled = mockPreemptionStatus("root");
     when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
+    QueueResourceQuotas rootQr = new QueueResourceQuotas();
+    rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0));
+    rootQr.setEffectiveMinResource(abs[0]);
+    rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+        Resource.newInstance(maxCap[0], 0));
+    rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]);
+    when(root.getQueueResourceQuotas()).thenReturn(rootQr);
+    when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+        .thenReturn(abs[0]);
+    when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+        .thenReturn(Resource.newInstance(maxCap[0], 0));
 
     for (int i = 1; i < queues.length; ++i) {
       final CSQueue q;
@@ -1234,6 +1246,18 @@ public class TestProportionalCapacityPreemptionPolicy {
       qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
       when(q.getQueueCapacities()).thenReturn(qc);
 
+      QueueResourceQuotas qr = new QueueResourceQuotas();
+      qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0));
+      qr.setEffectiveMinResource(abs[i]);
+      qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+          Resource.newInstance(maxCap[i], 0));
+      qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]);
+      when(q.getQueueResourceQuotas()).thenReturn(qr);
+      when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+          .thenReturn(abs[i]);
+      when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+          .thenReturn(Resource.newInstance(maxCap[i], 0));
+
       String parentPathName = p.getQueuePath();
       parentPathName = (parentPathName == null) ? "root" : parentPathName;
       String queuePathName = (parentPathName + "." + queueName).replace("/",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
index 7784549..a1d89d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
@@ -67,9 +67,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
     conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
         "priority_first");
 
-    String labelsConfig = "=100:200,true;";
+    String labelsConfig = "=100:50,true;";
     String nodesConfig = // n1 has no label
-        "n1= res=100:200";
+        "n1= res=100:50";
     String queuesConfig =
         // guaranteed,max,used,pending,reserved
         "root(=[100:50 100:50 80:40 120:60 0]);" + // root
@@ -105,7 +105,7 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
     verify(mDisp, times(1)).handle(argThat(
         new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
             getAppAttemptId(4))));
-    verify(mDisp, times(7)).handle(argThat(
+    verify(mDisp, times(3)).handle(argThat(
         new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
             getAppAttemptId(3))));
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
new file mode 100644
index 0000000..5a66281
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -0,0 +1,516 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAbsoluteResourceConfiguration {
+
+  private static final int GB = 1024;
+
+  private static final String QUEUEA = "queueA";
+  private static final String QUEUEB = "queueB";
+  private static final String QUEUEC = "queueC";
+  private static final String QUEUEA1 = "queueA1";
+  private static final String QUEUEA2 = "queueA2";
+  private static final String QUEUEB1 = "queueB1";
+
+  private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
+      + "." + QUEUEA;
+  private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
+      + "." + QUEUEB;
+  private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
+      + "." + QUEUEC;
+  private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
+  private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
+  private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+
+  private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
+      10);
+  private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB,
+      30);
+  private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB,
+      5);
+  private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB,
+      5);
+  private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB,
+      10);
+  private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB,
+      10);
+  private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB,
+      30);
+  private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB,
+      10);
+  private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB,
+      20);
+  private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6);
+  private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6);
+  private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6);
+  private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000,
+      20);
+
+  private static Set<String> resourceTypes = new HashSet<>(
+      Arrays.asList("memory", "vcores"));
+
+  private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
+      boolean isCapacityNeeded) {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+    // Set default capacities like normal configuration.
+    if (isCapacityNeeded) {
+      csConf.setCapacity(QUEUEA_FULL, 50f);
+      csConf.setCapacity(QUEUEB_FULL, 25f);
+      csConf.setCapacity(QUEUEC_FULL, 25f);
+    }
+
+    return csConf;
+  }
+
+  private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
+      boolean isCapacityNeeded) {
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[]{QUEUEA, QUEUEB, QUEUEC});
+    csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+    csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+
+    // Set default capacities like normal configuration.
+    if (isCapacityNeeded) {
+      csConf.setCapacity(QUEUEA_FULL, 50f);
+      csConf.setCapacity(QUEUEB_FULL, 25f);
+      csConf.setCapacity(QUEUEC_FULL, 25f);
+      csConf.setCapacity(QUEUEA1_FULL, 50f);
+      csConf.setCapacity(QUEUEA2_FULL, 50f);
+      csConf.setCapacity(QUEUEB1_FULL, 100f);
+    }
+
+    return csConf;
+  }
+
+  private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
+      CapacitySchedulerConfiguration csConf) {
+    // Update min/max resource to queueA/B/C
+    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+
+    csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+    csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+    csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+    return csConf;
+  }
+
+  private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig(
+      CapacitySchedulerConfiguration csConf) {
+    // Update min/max resource to queueA/B/C
+    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES);
+    csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+    csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+    csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+    csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+    return csConf;
+  }
+
+  @Test
+  public void testSimpleMinMaxResourceConfigurartionPerQueue() {
+
+    CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true);
+    setupMinMaxResourceConfiguration(csConf);
+
+    Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+        QUEUE_A_MINRES,
+        csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+    Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+        QUEUE_A_MAXRES,
+        csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+    Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+        QUEUE_B_MINRES,
+        csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+    Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+        QUEUE_B_MAXRES,
+        csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES,
+        csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES,
+        csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+  }
+
+  @Test
+  public void testEffectiveMinMaxResourceConfigurartionPerQueue()
+      throws Exception {
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+        false);
+    setupMinMaxResourceConfiguration(csConf);
+
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+    Assert.assertNotNull(qA);
+    Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+    Assert.assertNotNull(qB);
+    Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+    Assert.assertNotNull(qC);
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    rm.stop();
+  }
+
+  @Test
+  public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
+    /**
+     * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+     *
+     * Test below cases 1) Configure percentage based capacity and absolute
+     * resource together. 2) As per above tree structure, ensure all values
+     * could be retrieved. 3) Validate whether min resource cannot be more than
+     * max resources. 4) Validate whether max resource of queue cannot be more
+     * than its parent max resource.
+     */
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+        false);
+    setupMinMaxResourceConfiguration(csConf);
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    // 1. Create a new config with capcity and min/max together. Ensure an
+    // exception is thrown.
+    CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
+        true);
+    setupMinMaxResourceConfiguration(csConf1);
+
+    try {
+      cs.reinitialize(csConf1, rm.getRMContext());
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e instanceof IOException);
+      Assert.assertEquals(
+          "Failed to re-init queues : Queue 'queueA' should use either"
+              + " percentage based capacity configuration or absolute resource.",
+          e.getMessage());
+    }
+    rm.stop();
+
+    // 2. Create a new config with min/max alone with a complex queue config.
+    // Check all values could be fetched correctly.
+    CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf2);
+
+    rm = new MockRM(csConf2);
+    rm.start();
+    rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+    cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+    Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+    Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+    Assert.assertNotNull(qB1);
+    Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+    Assert.assertNotNull(qC);
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    // 3. Create a new config and make sure one queue's min resource is more
+    // than its max resource configured.
+    CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf3);
+
+    csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES);
+    csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+    try {
+      cs.reinitialize(csConf3, rm.getRMContext());
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e instanceof IOException);
+      Assert.assertEquals(
+          "Failed to re-init queues : Min resource configuration "
+              + "<memory:153600, vCores:30> is greater than its "
+              + "max value:<memory:40960, vCores:10> in queue:queueB1",
+          e.getMessage());
+    }
+
+    // 4. Create a new config and make sure one queue's max resource is more
+    // than its preant's max resource configured.
+    CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf4);
+
+    csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES);
+
+    try {
+      cs.reinitialize(csConf4, rm.getRMContext());
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e instanceof IOException);
+      Assert
+          .assertEquals(
+              "Failed to re-init queues : Max resource configuration "
+                  + "<memory:204800, vCores:30> is greater than parents max value:"
+                  + "<memory:153600, vCores:30> in queue:queueB1",
+              e.getMessage());
+    }
+    rm.stop();
+  }
+
+  @Test
+  public void testComplexValidateAbsoluteResourceConfig() throws Exception {
+    /**
+     * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+     *
+     * Test below cases: 1) Parent and its child queues must use either
+     * percentage based or absolute resource configuration. 2) Parent's min
+     * resource must be more than sum of child's min resource.
+     */
+
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf);
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+    // 1. Explicitly set percentage based config for parent queues. This will
+    // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
+    // resource.
+    csConf.setCapacity(QUEUEA_FULL, 50f);
+    csConf.setCapacity(QUEUEB_FULL, 25f);
+    csConf.setCapacity(QUEUEC_FULL, 25f);
+
+    // Also unset resource based config.
+    csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
+    csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
+    csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+    try {
+      cs.reinitialize(csConf, rm.getRMContext());
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e instanceof IOException);
+      Assert.assertEquals(
+          "Failed to re-init queues : Parent queue 'queueA' "
+              + "and child queue 'queueA1' should use either percentage based"
+              + " capacity configuration or absolute resource together.",
+          e.getMessage());
+    }
+
+    // 2. Create a new config and make sure one queue's min resource is more
+    // than its max resource configured.
+    CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf1);
+
+    // Configure QueueA with lesser resource than its children.
+    csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES);
+
+    try {
+      cs.reinitialize(csConf1, rm.getRMContext());
+      Assert.fail();
+    } catch (IOException e) {
+      Assert.assertTrue(e instanceof IOException);
+      Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: "
+          + "<memory:51200, vCores:5> is less than to its children:"
+          + "<memory:102400, vCores:10> for queue:queueA", e.getMessage());
+    }
+  }
+
+  @Test
+  public void testEffectiveResourceAfterReducingClusterResource()
+      throws Exception {
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+        false);
+    setupMinMaxResourceConfiguration(csConf);
+
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+    rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+    Assert.assertNotNull(qA);
+    Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+    Assert.assertNotNull(qB);
+    Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+    Assert.assertNotNull(qC);
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    // unregister one NM.
+    rm.unRegisterNode(nm1);
+
+    // After loosing one NM, effective min res of queueA will become just
+    // above half. Hence A's min will be 60Gi and 6 cores and max will be
+    // 128GB and 20 cores.
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    rm.stop();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 8aca235..24ae244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -86,6 +87,7 @@ public class TestApplicationLimits {
   final static int GB = 1024;
 
   LeafQueue queue;
+  CSQueue root;
   
   private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
 
@@ -100,7 +102,7 @@ public class TestApplicationLimits {
     setupQueueConfiguration(csConf);
     
     rmContext = TestUtils.getMockRMContext();
-
+    Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
 
     CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
     when(csContext.getConfiguration()).thenReturn(csConf);
@@ -110,10 +112,11 @@ public class TestApplicationLimits {
     when(csContext.getMaximumResourceCapability()).
         thenReturn(Resources.createResource(16*GB, 32));
     when(csContext.getClusterResource()).
-        thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
+        thenReturn(clusterResource);
     when(csContext.getResourceCalculator()).
         thenReturn(resourceCalculator);
     when(csContext.getRMContext()).thenReturn(rmContext);
+    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
     
     RMContainerTokenSecretManager containerTokenSecretManager =
         new RMContainerTokenSecretManager(conf);
@@ -122,13 +125,17 @@ public class TestApplicationLimits {
         containerTokenSecretManager);
 
     Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
-    CSQueue root = CapacitySchedulerQueueManager
+    root = CapacitySchedulerQueueManager
         .parseQueue(csContext, csConf, null, "root",
             queues, queues,
             TestUtils.spyHook);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
-    
     queue = spy(new LeafQueue(csContext, A, root, null));
+    QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
+        .getQueueResourceQuotas();
+    doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
 
     // Stub out ACL checks
     doReturn(true).
@@ -189,6 +196,8 @@ public class TestApplicationLimits {
     // when there is only 1 user, and drops to 2G (the userlimit) when there
     // is a second user
     Resource clusterResource = Resource.newInstance(80 * GB, 40);
+    root.updateClusterResource(clusterResource, new ResourceLimits(
+        clusterResource));
     queue.updateClusterResource(clusterResource, new ResourceLimits(
         clusterResource));
     
@@ -287,6 +296,8 @@ public class TestApplicationLimits {
     CSQueue root = 
         CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
             "root", queues, queues, TestUtils.spyHook);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     LeafQueue queue = (LeafQueue)queues.get(A);
     
@@ -357,6 +368,8 @@ public class TestApplicationLimits {
         csContext, csConf, null, "root",
         queues, queues, TestUtils.spyHook);
     clusterResource = Resources.createResource(100 * 16 * GB);
+    root.updateClusterResource(clusterResource, new ResourceLimits(
+        clusterResource));
 
     queue = (LeafQueue)queues.get(A);
 
@@ -378,6 +391,8 @@ public class TestApplicationLimits {
     root = CapacitySchedulerQueueManager.parseQueue(
         csContext, csConf, null, "root",
         queues, queues, TestUtils.spyHook);
+    root.updateClusterResource(clusterResource, new ResourceLimits(
+        clusterResource));
 
     queue = (LeafQueue)queues.get(A);
     assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
@@ -393,7 +408,7 @@ public class TestApplicationLimits {
     final String user_0 = "user_0";
     final String user_1 = "user_1";
     final String user_2 = "user_2";
-    
+
     assertEquals(Resource.newInstance(16 * GB, 1),
         queue.calculateAndGetAMResourceLimit());
     assertEquals(Resource.newInstance(8 * GB, 1),
@@ -578,6 +593,7 @@ public class TestApplicationLimits {
         thenReturn(Resources.createResource(16*GB));
     when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
     when(csContext.getRMContext()).thenReturn(rmContext);
+    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
     
     // Say cluster has 100 nodes of 16G each
     Resource clusterResource = Resources.createResource(100 * 16 * GB);
@@ -586,6 +602,8 @@ public class TestApplicationLimits {
     Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
     CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
         csConf, null, "root", queues, queues, TestUtils.spyHook);
+    rootQueue.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
     when(csContext.getClusterResourceUsage())
@@ -693,6 +711,8 @@ public class TestApplicationLimits {
 
     // Now reduce cluster size and check for the smaller headroom
     clusterResource = Resources.createResource(90*16*GB);
+    rootQueue.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Any change is cluster resource needs to enforce user-limit recomputation.
     // In existing code, LeafQueue#updateClusterResource handled this. However

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 0aac2ef..d73f1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -600,6 +601,7 @@ public class TestApplicationLimitsByPartition {
     RMContext spyRMContext = spy(rmContext);
     when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
     when(csContext.getRMContext()).thenReturn(spyRMContext);
+    when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
 
     mgr.activateNode(NodeId.newInstance("h0", 0),
         Resource.newInstance(160 * GB, 16)); // default Label
@@ -615,6 +617,8 @@ public class TestApplicationLimitsByPartition {
     Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
     CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
         csConf, null, "root", queues, queues, TestUtils.spyHook);
+    rootQueue.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
     when(csContext.getClusterResourceUsage())

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index e91f734..23a371a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4307,7 +4307,7 @@ public class TestCapacityScheduler {
           null, null, NULL_UPDATE_REQUESTS);
       CapacityScheduler.schedule(cs);
     }
-    assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
+    assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
         cs.getQueue("p2").getUsedResources().getMemorySize());
 
     //Free a container from X1

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 0fcc86d..fcfa0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -242,6 +242,8 @@ public class TestChildQueueOrder {
       Resources.createResource(numNodes * (memoryPerNode*GB), 
           numNodes * coresPerNode);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Start testing
     CSQueue a = queues.get(A);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index a32352b..96f8970 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -191,6 +191,8 @@ public class TestLeafQueue {
             CapacitySchedulerConfiguration.ROOT, 
             queues, queues, 
             TestUtils.spyHook);
+    root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
+        new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32)));
 
     ResourceUsage queueResUsage = root.getQueueResourceUsage();
     when(csContext.getClusterResourceUsage())
@@ -307,13 +309,11 @@ public class TestLeafQueue {
     // Verify the value for getAMResourceLimit for queues with < .1 maxcap
     Resource clusterResource = Resource.newInstance(50 * GB, 50);
 
-    a.updateClusterResource(clusterResource,
+    root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
     assertEquals(Resource.newInstance(1 * GB, 1),
         a.calculateAndGetAMResourceLimit());
 
-    b.updateClusterResource(clusterResource,
-        new ResourceLimits(clusterResource));
     assertEquals(Resource.newInstance(5 * GB, 1),
         b.calculateAndGetAMResourceLimit());
   }
@@ -358,6 +358,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
@@ -556,6 +558,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
@@ -630,6 +634,8 @@ public class TestLeafQueue {
     // Test max-capacity
     // Now - no more allocs since we are at max-cap
     a.setMaxCapacity(0.5f);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     applyCSAssignment(clusterResource,
         a.assignContainers(clusterResource, node_0,
         new ResourceLimits(clusterResource),
@@ -699,6 +705,8 @@ public class TestLeafQueue {
     Resource clusterResource =
         Resources.createResource(numNodes * (80 * GB), numNodes * 100);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Set user-limit. Need a small queue within a large cluster.
     b.setUserLimit(50);
@@ -779,6 +787,8 @@ public class TestLeafQueue {
         Resources.createResource(numNodes * (8 * GB), numNodes * 100);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
     when(csContext.getClusterResource()).thenReturn(clusterResource);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests so that one application is memory dominant
     // and other application is vcores dominant
@@ -891,6 +901,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
  
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
@@ -915,6 +927,8 @@ public class TestLeafQueue {
     // Set user-limit
     a.setUserLimit(50);
     a.setUserLimitFactor(2);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     
     // There're two active users
     assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers());
@@ -940,7 +954,7 @@ public class TestLeafQueue {
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
 
     // Allocate one container to app_0, before allocating this container,
-    // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <=
+    // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <=
     // user-limit.
     applyCSAssignment(clusterResource,
         a.assignContainers(clusterResource, node_1,
@@ -1068,15 +1082,9 @@ public class TestLeafQueue {
         a.assignContainers(clusterResource, node_1,
         new ResourceLimits(clusterResource),
         SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
-    assertEquals(9*GB, a.getUsedResources().getMemorySize());
-    assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
-    assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-
-    assertEquals(4*GB,
-        app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
-
-    assertEquals(1*GB,
-        app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+    assertEquals(12*GB, a.getUsedResources().getMemorySize());
+    assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
+    assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
   }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1100,6 +1108,8 @@ public class TestLeafQueue {
     final int numNodes = 2;
     Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager
         = mock(CapacitySchedulerQueueManager.class);
@@ -1122,6 +1132,8 @@ public class TestLeafQueue {
     qb.setUserLimit(100);
     qb.setUserLimitFactor(1);
 
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     final ApplicationAttemptId appAttemptId_0 =
               TestUtils.getMockApplicationAttemptId(0, 0);
     FiCaSchedulerApp app_0 =
@@ -1256,106 +1268,6 @@ public class TestLeafQueue {
   }
 
   @Test
-  public void testUserHeadroomMultiApp() throws Exception {
-    // Mock the queue
-    LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
-    //unset maxCapacity
-    a.setMaxCapacity(1.0f);
-
-    // Users
-    final String user_0 = "user_0";
-    final String user_1 = "user_1";
-
-    // Submit applications
-    final ApplicationAttemptId appAttemptId_0 =
-        TestUtils.getMockApplicationAttemptId(0, 0);
-    FiCaSchedulerApp app_0 =
-        new FiCaSchedulerApp(appAttemptId_0, user_0, a,
-            a.getAbstractUsersManager(), spyRMContext);
-    a.submitApplicationAttempt(app_0, user_0);
-
-    final ApplicationAttemptId appAttemptId_1 =
-        TestUtils.getMockApplicationAttemptId(1, 0);
-    FiCaSchedulerApp app_1 =
-        new FiCaSchedulerApp(appAttemptId_1, user_0, a,
-            a.getAbstractUsersManager(), spyRMContext);
-    a.submitApplicationAttempt(app_1, user_0);  // same user
-
-    final ApplicationAttemptId appAttemptId_2 =
-        TestUtils.getMockApplicationAttemptId(2, 0);
-    FiCaSchedulerApp app_2 =
-        new FiCaSchedulerApp(appAttemptId_2, user_1, a,
-            a.getAbstractUsersManager(), spyRMContext);
-    a.submitApplicationAttempt(app_2, user_1);
-
-    // Setup some nodes
-    String host_0 = "127.0.0.1";
-    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 
-      0, 16*GB);
-    String host_1 = "127.0.0.2";
-    FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 
-      0, 16*GB);
-
-    Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
-        app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
-        app_1, app_2.getApplicationAttemptId(), app_2);
-    Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
-        node_0, node_1.getNodeID(), node_1);
-
-    final int numNodes = 2;
-    Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1);
-    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
-    Priority priority = TestUtils.createMockPriority(1);
-
-    app_0.updateResourceRequests(Collections.singletonList(
-            TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
-                priority, recordFactory)));
-
-    applyCSAssignment(clusterResource,
-        a.assignContainers(clusterResource, node_0,
-        new ResourceLimits(clusterResource),
-        SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
-    assertEquals(1*GB, a.getUsedResources().getMemorySize());
-    assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
-    assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
-    //Now, headroom is the same for all apps for a given user + queue combo
-    //and a change to any app's headroom is reflected for all the user's apps
-    //once those apps are active/have themselves calculated headroom for 
-    //allocation at least one time
-    assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
-    assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active
-    assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
-    app_1.updateResourceRequests(Collections.singletonList(
-        TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
-            priority, recordFactory)));
-
-    applyCSAssignment(clusterResource,
-        a.assignContainers(clusterResource, node_0,
-        new ResourceLimits(clusterResource),
-        SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
-    assertEquals(2*GB, a.getUsedResources().getMemorySize());
-    assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
-    assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-    assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
-    assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active
-    assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
-    //Complete container and verify that headroom is updated, for both apps 
-    //for the user
-    RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
-    a.completedContainer(clusterResource, app_0, node_0, rmContainer,
-    ContainerStatus.newInstance(rmContainer.getContainerId(),
-	ContainerState.COMPLETE, "",
-	ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
-    RMContainerEventType.KILL, null, true);
-
-    assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
-    assertEquals(2*GB, app_1.getHeadroom().getMemorySize());
-  }
-
-  @Test
   public void testHeadroomWithMaxCap() throws Exception {
     // Mock the queue
     LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
@@ -1403,7 +1315,12 @@ public class TestLeafQueue {
     final int numNodes = 2;
     Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
- 
+
+    ParentQueue root = (ParentQueue) queues
+        .get(CapacitySchedulerConfiguration.ROOT);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
     app_0.updateResourceRequests(Collections.singletonList(
@@ -1454,6 +1371,8 @@ public class TestLeafQueue {
     
     // Submit requests for app_1 and set max-cap
     a.setMaxCapacity(.1f);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     app_2.updateResourceRequests(Collections.singletonList(
         TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
             priority, recordFactory)));
@@ -1542,6 +1461,8 @@ public class TestLeafQueue {
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
     when(csContext.getClusterResource()).thenReturn(clusterResource);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
@@ -1624,6 +1545,8 @@ public class TestLeafQueue {
     // Test max-capacity
     // Now - no more allocs since we are at max-cap
     a.setMaxCapacity(0.5f);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     applyCSAssignment(clusterResource,
         a.assignContainers(clusterResource, node_0,
         new ResourceLimits(clusterResource),
@@ -1638,6 +1561,8 @@ public class TestLeafQueue {
     // Now, allocations should goto app_3 since it's under user-limit 
     a.setMaxCapacity(1.0f);
     a.setUserLimitFactor(1);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     applyCSAssignment(clusterResource,
         a.assignContainers(clusterResource, node_0,
         new ResourceLimits(clusterResource),
@@ -1743,6 +1668,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (4*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
@@ -1880,6 +1807,8 @@ public class TestLeafQueue {
     final int numNodes = 3;
     Resource clusterResource = 
         Resources.createResource(numNodes * (4*GB), numNodes * 16);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
     when(csContext.getMaximumResourceCapability()).thenReturn(
         Resources.createResource(4*GB, 16));
@@ -2051,6 +1980,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     
     // Setup resource-requests and submit
     Priority priority = TestUtils.createMockPriority(1);
@@ -2237,11 +2168,10 @@ public class TestLeafQueue {
     CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
         csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
         TestUtils.spyHook);
-    queues = newQueues;
     root.reinitialize(newRoot, cs.getClusterResource());
 
     // Manipulate queue 'b'
-    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
+    LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B));
 
     // Check locality parameters.
     assertEquals(2, a.getNodeLocalityDelay());
@@ -2277,6 +2207,8 @@ public class TestLeafQueue {
     Resource clusterResource =
         Resources.createResource(numNodes * (8 * GB), numNodes * 16);
     when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes);
+    newRoot.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests and submit
     Priority priority = TestUtils.createMockPriority(1);
@@ -2412,6 +2344,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), 1);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     
     // Setup resource-requests and submit
     List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
@@ -2545,6 +2479,8 @@ public class TestLeafQueue {
     Resource clusterResource = Resources.createResource(
         numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests and submit
     Priority priority = TestUtils.createMockPriority(1);
@@ -2660,17 +2596,14 @@ public class TestLeafQueue {
     assertEquals(2, e.getNumActiveApplications());
     assertEquals(1, e.getNumPendingApplications());
 
-    csConf.setDouble(CapacitySchedulerConfiguration
-        .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
-        CapacitySchedulerConfiguration
-        .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
+    csConf.setDouble(
+        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+        CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
+            * 2);
     Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
-    CSQueue newRoot =
-        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
-            CapacitySchedulerConfiguration.ROOT,
-            newQueues, queues,
-            TestUtils.spyHook);
-    queues = newQueues;
+    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+        csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+        TestUtils.spyHook);
     root.reinitialize(newRoot, csContext.getClusterResource());
 
     // after reinitialization
@@ -2697,7 +2630,6 @@ public class TestLeafQueue {
             CapacitySchedulerConfiguration.ROOT,
             newQueues, queues,
             TestUtils.spyHook);
-    queues = newQueues;
     root.reinitialize(newRoot, cs.getClusterResource());
 
     // after reinitialization
@@ -2745,7 +2677,7 @@ public class TestLeafQueue {
     assertEquals(1, e.getNumPendingApplications());
 
     Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32); 
-    e.updateClusterResource(clusterResource,
+    root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
     // after updating cluster resource
@@ -2837,6 +2769,9 @@ public class TestLeafQueue {
         numNodes * (8*GB), numNodes * 1);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
 
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
     // Setup resource-requests
     // resourceName: <priority, memory, #containers, relaxLocality>
     // host_0_0: < 1, 1GB, 1, true >
@@ -3036,36 +2971,44 @@ public class TestLeafQueue {
   @Test
   public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
       throws Exception {
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
     CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
-    Resource clusterResource = Resources
-        .createResource(100 * 16 * GB, 100 * 32);
+    final String newRootName = "root" + System.currentTimeMillis();
+    setupQueueConfiguration(csConf, newRootName);
+
+    Resource clusterResource = Resources.createResource(100 * 16 * GB,
+        100 * 32);
     CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
     when(csContext.getRMContext()).thenReturn(rmContext);
-    csConf.setFloat(CapacitySchedulerConfiguration.
-        MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
-    ParentQueue root = new ParentQueue(csContext, 
-        CapacitySchedulerConfiguration.ROOT, null, null);
-    csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
-    LeafQueue a = new LeafQueue(csContext, A, root, null);
-    assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
-    assertEquals(a.calculateAndGetAMResourceLimit(),
-        Resources.createResource(160 * GB, 1));
-    
-    csConf.setFloat(CapacitySchedulerConfiguration.
-        MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
-    LeafQueue newA = new LeafQueue(csContext, A, root, null);
-    a.reinitialize(newA, clusterResource);
-    assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
-    assertEquals(a.calculateAndGetAMResourceLimit(),
-        Resources.createResource(320 * GB, 1));
+    csConf.setFloat(
+        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+        0.1f);
+
+    CSQueue root;
+    root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+        CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
-    Resource newClusterResource = Resources.createResource(100 * 20 * GB,
-        100 * 32);
-    a.updateClusterResource(newClusterResource, 
-        new ResourceLimits(newClusterResource));
-    //  100 * 20 * 0.2 = 400
-    assertEquals(a.calculateAndGetAMResourceLimit(),
-        Resources.createResource(400 * GB, 1));
+    // Manipulate queue 'a'
+    LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B));
+    assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f);
+    assertEquals(b.calculateAndGetAMResourceLimit(),
+        Resources.createResource(159 * GB, 1));
+
+    csConf.setFloat(
+        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+        0.2f);
+    clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32);
+    Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+        csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+        TestUtils.spyHook);
+    root.reinitialize(newRoot, clusterResource);
+
+    b = stubLeafQueue((LeafQueue) newQueues.get(B));
+    assertEquals(b.calculateAndGetAMResourceLimit(),
+        Resources.createResource(320 * GB, 1));
   }
   
   @Test
@@ -3142,6 +3085,8 @@ public class TestLeafQueue {
     Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
         numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     String user_0 = "user_0";
 
@@ -3308,6 +3253,8 @@ public class TestLeafQueue {
     Resource clusterResource = Resources.createResource(
         numNodes * (16*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     String user_0 = "user_0";
 
@@ -3435,6 +3382,8 @@ public class TestLeafQueue {
     Resource clusterResource = 
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
     
     // Setup resource-requests and submit
     // App0 has node local request for host_0/host_1, and app1 has node local
@@ -3533,6 +3482,8 @@ public class TestLeafQueue {
     Resource clusterResource =
         Resources.createResource(numNodes * (100*GB), numNodes * 128);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Pending resource requests for app_0 and app_1 total 5GB.
     Priority priority = TestUtils.createMockPriority(1);
@@ -3699,6 +3650,8 @@ public class TestLeafQueue {
     Resource clusterResource =
         Resources.createResource(numNodes * (100*GB), numNodes * 128);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Pending resource requests for user_0: app_0 and app_1 total 3GB.
     Priority priority = TestUtils.createMockPriority(1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index a9196d1..25a9774 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -249,6 +249,8 @@ public class TestParentQueue {
         Resources.createResource(numNodes * (memoryPerNode*GB),
             numNodes * coresPerNode);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Start testing
     LeafQueue a = (LeafQueue)queues.get(A);
@@ -494,6 +496,8 @@ public class TestParentQueue {
         Resources.createResource(numNodes * (memoryPerNode*GB), 
             numNodes * coresPerNode);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Start testing
     CSQueue a = queues.get(A);
@@ -710,6 +714,8 @@ public class TestParentQueue {
         Resources.createResource(numNodes * (memoryPerNode*GB),
             numNodes * coresPerNode);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Start testing
     LeafQueue a = (LeafQueue)queues.get(A);
@@ -790,6 +796,8 @@ public class TestParentQueue {
         Resources.createResource(numNodes * (memoryPerNode*GB),
             numNodes * coresPerNode);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Start testing
     LeafQueue b3 = (LeafQueue)queues.get(B3);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 32f022f..0855720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -267,6 +267,8 @@ public class TestReservations {
     final int numNodes = 3;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -454,6 +456,8 @@ public class TestReservations {
     final int numNodes = 3;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -600,6 +604,8 @@ public class TestReservations {
     final int numNodes = 3;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -782,6 +788,8 @@ public class TestReservations {
     final int numNodes = 2;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -896,6 +904,10 @@ public class TestReservations {
     String host_1 = "host_1";
     FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
         8 * GB);
+    
+    Resource clusterResource = Resources.createResource(2 * 8 * GB);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
 
     // Setup resource-requests
@@ -1068,6 +1080,8 @@ public class TestReservations {
     final int numNodes = 2;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1256,6 +1270,8 @@ public class TestReservations {
     final int numNodes = 2;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1418,6 +1434,9 @@ public class TestReservations {
     final int numNodes = 3;
     Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
 
     // Setup resource-requests
     Priority priorityAM = TestUtils.createMockPriority(1);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[34/50] hadoop git commit: YARN-7520. Queue Ordering policy changes for ordering auto created leaf queues within Managed parent Queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
YARN-7520. Queue Ordering policy changes for ordering auto created leaf queues within Managed parent Queues. (Suma Shivaprasad via wangda)

Change-Id: I482f086945bd448d512cb5b3879d7371e37ee134


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8316df8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8316df8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8316df8

Branch: refs/heads/HDFS-7240
Commit: a8316df8c05a7b3d1a5577174b838711a49ef971
Parents: f548bff
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:11:28 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:11:28 2017 -0800

----------------------------------------------------------------------
 .../PriorityUtilizationQueueOrderingPolicy.java | 83 ++++++++++++++------
 ...tPriorityUtilizationQueueOrderingPolicy.java | 78 ++++++++++++------
 2 files changed, 113 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8316df8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 4985a1a..e684c2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,9 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels
+    .RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
@@ -49,7 +51,8 @@ import java.util.function.Supplier;
  *   other is under: The queue that is under its capacity guarantee gets the
  *   resources.
  */
-public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPolicy {
+public class PriorityUtilizationQueueOrderingPolicy
+    implements QueueOrderingPolicy {
   private List<CSQueue> queues;
   private boolean respectPriority;
 
@@ -78,7 +81,7 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
     if (priority1 == priority2) {
       // The queue with less relative used-capacity goes first
       return Double.compare(relativeAssigned1, relativeAssigned2);
-    } else {
+    } else{
       // When priority is different:
       if ((relativeAssigned1 < 1.0f && relativeAssigned2 < 1.0f) || (
           relativeAssigned1 >= 1.0f && relativeAssigned2 >= 1.0f)) {
@@ -86,7 +89,7 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
         // Or both the queues are over or meeting their guaranteed capacities
         // queue with higher used-capacity goes first
         return Integer.compare(priority2, priority1);
-      } else {
+      } else{
         // Otherwise, when one of the queues is over or meeting their
         // guaranteed capacities and the other is under: The queue that is
         // under its capacity guarantee gets the resources.
@@ -98,7 +101,7 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
   /**
    * Comparator that both looks at priority and utilization
    */
-  private class PriorityQueueComparator implements  Comparator<CSQueue> {
+  private class PriorityQueueComparator implements Comparator<CSQueue> {
 
     @Override
     public int compare(CSQueue q1, CSQueue q2) {
@@ -109,8 +112,38 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
         return rc;
       }
 
-      float used1 = q1.getQueueCapacities().getUsedCapacity(p);
-      float used2 = q2.getQueueCapacities().getUsedCapacity(p);
+      float q1AbsCapacity = q1.getQueueCapacities().getAbsoluteCapacity(p);
+      float q2AbsCapacity = q2.getQueueCapacities().getAbsoluteCapacity(p);
+
+      //If q1's abs capacity > 0 and q2 is 0, then prioritize q1
+      if (Float.compare(q1AbsCapacity, 0f) > 0 && Float.compare(q2AbsCapacity,
+          0f) == 0) {
+        return -1;
+        //If q2's abs capacity > 0 and q1 is 0, then prioritize q2
+      } else if (Float.compare(q2AbsCapacity, 0f) > 0 && Float.compare(
+          q1AbsCapacity, 0f) == 0) {
+        return 1;
+      } else if (Float.compare(q1AbsCapacity, 0f) == 0 && Float.compare(
+          q2AbsCapacity, 0f) == 0) {
+        // both q1 has 0 and q2 has 0 capacity, then fall back to using
+        // priority, abs used capacity to prioritize
+        float used1 = q1.getQueueCapacities().getAbsoluteUsedCapacity(p);
+        float used2 = q2.getQueueCapacities().getAbsoluteUsedCapacity(p);
+
+        return compare(q1, q2, used1, used2, p);
+      } else{
+        // both q1 has positive abs capacity and q2 has positive abs
+        // capacity
+        float used1 = q1.getQueueCapacities().getUsedCapacity(p);
+        float used2 = q2.getQueueCapacities().getUsedCapacity(p);
+
+        return compare(q1, q2, used1, used2, p);
+      }
+    }
+
+    private int compare(CSQueue q1, CSQueue q2, float q1Used, float q2Used,
+        String partition) {
+
       int p1 = 0;
       int p2 = 0;
       if (respectPriority) {
@@ -118,29 +151,31 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
         p2 = q2.getPriority().getPriority();
       }
 
-      rc = PriorityUtilizationQueueOrderingPolicy.compare(used1, used2, p1, p2);
+      int rc = PriorityUtilizationQueueOrderingPolicy.compare(q1Used, q2Used,
+          p1, p2);
 
       // For queue with same used ratio / priority, queue with higher configured
       // capacity goes first
       if (0 == rc) {
-        Resource minEffRes1 = q1.getQueueResourceQuotas()
-            .getConfiguredMinResource(p);
-        Resource minEffRes2 = q2.getQueueResourceQuotas()
-            .getConfiguredMinResource(p);
-        if (!minEffRes1.equals(Resources.none())
-            && !minEffRes2.equals(Resources.none())) {
+        Resource minEffRes1 =
+            q1.getQueueResourceQuotas().getConfiguredMinResource(partition);
+        Resource minEffRes2 =
+            q2.getQueueResourceQuotas().getConfiguredMinResource(partition);
+        if (!minEffRes1.equals(Resources.none()) && !minEffRes2.equals(
+            Resources.none())) {
           return minEffRes2.compareTo(minEffRes1);
         }
 
-        float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
-        float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
+        float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(partition);
+        float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(partition);
         return Float.compare(abs2, abs1);
       }
 
       return rc;
     }
 
-    private int compareQueueAccessToPartition(CSQueue q1, CSQueue q2, String partition) {
+    private int compareQueueAccessToPartition(CSQueue q1, CSQueue q2,
+        String partition) {
       // Everybody has access to default partition
       if (StringUtils.equals(partition, RMNodeLabelsManager.NO_LABEL)) {
         return 0;
@@ -190,9 +225,11 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
   @Override
   public String getConfigName() {
     if (respectPriority) {
-      return CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY;
+      return CapacitySchedulerConfiguration.
+          QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY;
     } else{
-      return CapacitySchedulerConfiguration.QUEUE_UTILIZATION_ORDERING_POLICY;
+      return CapacitySchedulerConfiguration.
+          QUEUE_UTILIZATION_ORDERING_POLICY;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8316df8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index b9d5b82..ca9a84b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -36,7 +36,7 @@ import static org.mockito.Mockito.when;
 
 public class TestPriorityUtilizationQueueOrderingPolicy {
   private List<CSQueue> mockCSQueues(String[] queueNames, int[] priorities,
-      float[] utilizations, String partition) {
+      float[] utilizations, float[] absCapacities, String partition) {
     // sanity check
     assert queueNames != null && priorities != null && utilizations != null
         && queueNames.length > 0 && queueNames.length == priorities.length
@@ -48,6 +48,7 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
       when(q.getQueueName()).thenReturn(queueNames[i]);
 
       QueueCapacities qc = new QueueCapacities(false);
+      qc.setAbsoluteCapacity(partition, absCapacities[i]);
       qc.setUsedCapacity(partition, utilizations[i]);
 
       when(q.getQueueCapacities()).thenReturn(qc);
@@ -81,41 +82,45 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
 
     // Case 1, one queue
     policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 0 },
-        new float[] { 0.1f }, ""));
+        new float[] { 0.1f }, new float[] {0.2f},  ""));
     verifyOrder(policy, "", new String[] { "a" });
 
     // Case 2, 2 queues
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 0, 0 },
-        new float[] { 0.1f, 0.0f }, ""));
+        new float[] { 0.1f, 0.0f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "b", "a" });
 
     // Case 3, 3 queues
     policy.setQueues(
         mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 0, 0, 0 },
-            new float[] { 0.1f, 0.0f, 0.2f }, ""));
+            new float[] { 0.1f, 0.0f, 0.2f }, new float[] {0.2f, 0.3f, 0.4f},
+            ""));
     verifyOrder(policy, "", new String[] { "b", "a", "c" });
 
     // Case 4, 3 queues, ignore priority
     policy.setQueues(
         mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
-            new float[] { 0.1f, 0.0f, 0.2f }, ""));
+            new float[] { 0.1f, 0.0f, 0.2f }, new float[] {0.2f, 0.3f, 0.4f},
+            ""));
     verifyOrder(policy, "", new String[] { "b", "a", "c" });
 
     // Case 5, 3 queues, look at partition (default)
     policy.setQueues(
         mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
-            new float[] { 0.1f, 0.0f, 0.2f }, "x"));
+            new float[] { 0.1f, 0.0f, 0.2f }, new float[] {0.2f, 0.3f, 0.4f},
+            "x"));
     verifyOrder(policy, "", new String[] { "a", "b", "c" });
 
     // Case 5, 3 queues, look at partition (x)
     policy.setQueues(
         mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
-            new float[] { 0.1f, 0.0f, 0.2f }, "x"));
+            new float[] { 0.1f, 0.0f, 0.2f }, new float[] {0.2f, 0.3f, 0.4f},
+            "x"));
     verifyOrder(policy, "x", new String[] { "b", "a", "c" });
 
     // Case 6, 3 queues, with different accessibility to partition
     List<CSQueue> queues = mockCSQueues(new String[] { "a", "b", "c" }, new int[] { 2, 1, 0 },
-        new float[] { 0.1f, 0.0f, 0.2f }, "x");
+        new float[] { 0.1f, 0.0f, 0.2f }, new float[] {0.2f, 0.3f, 0.4f},  "x");
     // a can access "x"
     when(queues.get(0).getAccessibleNodeLabels()).thenReturn(ImmutableSet.of("x", "y"));
     // c can access "x"
@@ -131,89 +136,94 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
 
     // Case 1, one queue
     policy.setQueues(mockCSQueues(new String[] { "a" }, new int[] { 1 },
-        new float[] { 0.1f }, ""));
+        new float[] { 0.1f }, new float[] {0.2f}, ""));
     verifyOrder(policy, "", new String[] { "a" });
 
     // Case 2, 2 queues, both under utilized, same priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
-        new float[] { 0.2f, 0.1f }, ""));
+        new float[] { 0.2f, 0.1f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "b", "a" });
 
     // Case 3, 2 queues, both over utilized, same priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
-        new float[] { 1.1f, 1.2f }, ""));
+        new float[] { 1.1f, 1.2f },new float[] {0.2f, 0.3f},  ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 4, 2 queues, one under and one over, same priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
-        new float[] { 0.1f, 1.2f }, ""));
+        new float[] { 0.1f, 1.2f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 5, 2 queues, both over utilized, different priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
-        new float[] { 1.1f, 1.2f }, ""));
+        new float[] { 1.1f, 1.2f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "b", "a" });
 
     // Case 6, 2 queues, both under utilized, different priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
-        new float[] { 0.1f, 0.2f }, ""));
+        new float[] { 0.1f, 0.2f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "b", "a" });
 
     // Case 7, 2 queues, one under utilized and one over utilized,
     // different priority (1)
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
-        new float[] { 0.1f, 1.2f }, ""));
+        new float[] { 0.1f, 1.2f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 8, 2 queues, one under utilized and one over utilized,
     // different priority (1)
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 },
-        new float[] { 0.1f, 1.2f }, ""));
+        new float[] { 0.1f, 1.2f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 9, 2 queues, one under utilized and one meet, different priority (1)
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
-        new float[] { 0.1f, 1.0f }, ""));
+        new float[] { 0.1f, 1.0f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 10, 2 queues, one under utilized and one meet, different priority (2)
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 2, 1 },
-        new float[] { 0.1f, 1.0f }, ""));
+        new float[] { 0.1f, 1.0f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 11, 2 queues, one under utilized and one meet, same priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
-        new float[] { 0.1f, 1.0f }, ""));
+        new float[] { 0.1f, 1.0f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "a", "b" });
 
     // Case 12, 2 queues, both meet, different priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 2 },
-        new float[] { 1.0f, 1.0f }, ""));
+        new float[] { 1.0f, 1.0f }, new float[] {0.2f, 0.3f}, ""));
     verifyOrder(policy, "", new String[] { "b", "a" });
 
     // Case 13, 5 queues, different priority
     policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
         new int[] { 1, 2, 0, 0, 3 },
-        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, ""));
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f },
+        new float[] { 0.2f, 0.1f, 0.1f, 0.3f, 0.3f }, ""));
     verifyOrder(policy, "", new String[] { "e", "c", "b", "a", "d" });
 
-    // Case 14, 5 queues, different priority, partition default;
+    // Case 14, 5 queues, different priority,
+    // partition default - abs capacity is 0;
     policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
         new int[] { 1, 2, 0, 0, 3 },
-        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x"));
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f },
+        new float[] { 0.2f, 0.1f, 0.1f, 0.3f, 0.3f }, "x"));
     verifyOrder(policy, "", new String[] { "e", "b", "a", "c", "d" });
 
     // Case 15, 5 queues, different priority, partition x;
     policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
         new int[] { 1, 2, 0, 0, 3 },
-        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x"));
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f },
+        new float[] { 0.2f, 0.1f, 0.1f, 0.3f, 0.3f }, "x"));
     verifyOrder(policy, "x", new String[] { "e", "c", "b", "a", "d" });
 
     // Case 16, 5 queues, different priority, partition x; and different
     // accessibility
     List<CSQueue> queues = mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
         new int[] { 1, 2, 0, 0, 3 },
-        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f }, "x");
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f },
+        new float[] { 0.2f, 0.1f, 0.1f, 0.3f, 0.3f }, "x");
     // Only a/d has access to x
     when(queues.get(0).getAccessibleNodeLabels()).thenReturn(
         ImmutableSet.of("x"));
@@ -221,5 +231,23 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
         ImmutableSet.of("x"));
     policy.setQueues(queues);
     verifyOrder(policy, "x", new String[] { "a", "d", "e", "c", "b" });
+
+    // Case 17, 2 queues, one's abs capacity is 0
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 0.1f, 1.2f }, new float[] {0.0f, 0.3f}, ""));
+    verifyOrder(policy, "", new String[] { "b", "a" });
+
+    // Case 18, 2 queues, one's abs capacity is 0
+    policy.setQueues(mockCSQueues(new String[] { "a", "b" }, new int[] { 1, 1 },
+        new float[] { 0.1f, 1.2f }, new float[] {0.3f, 0.0f}, ""));
+    verifyOrder(policy, "", new String[] { "a", "b" });
+
+    //Case 19, 5 queues with 2 having abs capacity 0 are prioritized last
+    policy.setQueues(mockCSQueues(new String[] { "a", "b", "c", "d", "e" },
+        new int[] { 1, 2, 0, 0, 3 },
+        new float[] { 1.2f, 1.0f, 0.2f, 1.1f, 0.2f },
+        new float[] { 0.0f, 0.0f, 0.1f, 0.3f, 0.3f }, "x"));
+    verifyOrder(policy, "x", new String[] { "e", "c", "d", "b", "a" });
+
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[35/50] hadoop git commit: YARN-7591. NPE in async-scheduling mode of CapacityScheduler. (Tao Yang via wangda)

Posted by xy...@apache.org.
YARN-7591. NPE in async-scheduling mode of CapacityScheduler. (Tao Yang via wangda)

Change-Id: I46689e530550ee0a6ac7a29786aab2cc1bdf314f


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adca1a72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adca1a72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adca1a72

Branch: refs/heads/HDFS-7240
Commit: adca1a72e4eca2ea634551e9fb8e9b878c36cb5c
Parents: a8316df
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:17:02 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:17:02 2017 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/CapacityScheduler.java   |  4 ++++
 .../scheduler/capacity/LeafQueue.java           | 21 +++++++++++++++++++-
 .../scheduler/common/fica/FiCaSchedulerApp.java |  5 +++++
 3 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/adca1a72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a5efd9f..8de3631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1301,6 +1301,10 @@ public class CapacityScheduler extends
     if (reservedContainer != null) {
       FiCaSchedulerApp reservedApplication = getCurrentAttemptForContainer(
           reservedContainer.getContainerId());
+      if (reservedApplication == null) {
+        LOG.error("Trying to schedule for a finished app, please double check.");
+        return null;
+      }
 
       // Try to fulfill the reservation
       LOG.info(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/adca1a72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 86fcbc9..ac1a26c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1201,7 +1201,14 @@ public class LeafQueue extends AbstractCSQueue {
             allocation.getSchedulingMode(), null);
 
         // Deduct resources that we can release
-        Resource usedResource = Resources.clone(getUser(username).getUsed(p));
+        User user = getUser(username);
+        if (user == null) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("User " + username + " has been removed!");
+          }
+          return false;
+        }
+        Resource usedResource = Resources.clone(user.getUsed(p));
         Resources.subtractFrom(usedResource,
             request.getTotalReleasedResource());
 
@@ -1406,6 +1413,12 @@ public class LeafQueue extends AbstractCSQueue {
       SchedulingMode schedulingMode, Resource userLimit) {
     String user = application.getUser();
     User queueUser = getUser(user);
+    if (queueUser == null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("User " + user + " has been removed!");
+      }
+      return Resources.none();
+    }
 
     // Compute user limit respect requested labels,
     // TODO, need consider headroom respect labels also
@@ -1500,6 +1513,12 @@ public class LeafQueue extends AbstractCSQueue {
     try {
       readLock.lock();
       User user = getUser(userName);
+      if (user == null) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("User " + userName + " has been removed!");
+        }
+        return false;
+      }
 
       currentResourceLimits.setAmountNeededUnreserve(Resources.none());
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/adca1a72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index a4e0096..776a7e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -322,6 +322,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
     RMContainer reservedContainerOnNode =
         schedulerContainer.getSchedulerNode().getReservedContainer();
     if (reservedContainerOnNode != null) {
+      // adding NP check as this proposal could not be allocated from reserved
+      // container in async-scheduling mode
+      if (allocation.getAllocateFromReservedContainer() == null) {
+        return false;
+      }
       RMContainer fromReservedContainer =
           allocation.getAllocateFromReservedContainer().getRmContainer();
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[50/50] hadoop git commit: Merge branch 'trunk' into HDFS-7240

Posted by xy...@apache.org.
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/070ad843
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/070ad843
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/070ad843

Branch: refs/heads/HDFS-7240
Commit: 070ad8438b29ae1044bb5cccc3aebcdeba93f95d
Parents: 4541ead 2abab1d
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Dec 12 15:54:07 2017 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Dec 12 15:54:07 2017 -0800

----------------------------------------------------------------------
 .../hadoop/fs/StreamCapabilitiesPolicy.java     |  15 +-
 .../io/erasurecode/ErasureCodeConstants.java    |   2 +-
 .../org/apache/hadoop/security/Credentials.java | 112 ++-
 .../hadoop/security/token/DtFileOperations.java |  10 +-
 .../org/apache/hadoop/util/StringUtils.java     |  31 +
 .../fs/contract/AbstractContractOpenTest.java   |  53 +-
 .../apache/hadoop/net/TestClusterTopology.java  |  58 +-
 .../hadoop/security/token/TestDtUtilShell.java  |  15 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  74 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |  28 +-
 .../hadoop/hdfs/protocol/ClientProtocol.java    |  15 +
 .../hadoop/hdfs/protocol/HdfsPathHandle.java    |  65 +-
 .../ClientNamenodeProtocolTranslatorPB.java     |  25 +-
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  |  15 -
 .../src/main/proto/ClientNamenodeProtocol.proto |  11 +
 .../src/main/proto/hdfs.proto                   |   2 +
 ...tNamenodeProtocolServerSideTranslatorPB.java |  22 +
 .../blockmanagement/InvalidateBlocks.java       |   8 +-
 .../hdfs/server/datanode/BlockReceiver.java     |   5 +
 .../federation/router/RouterRpcServer.java      |  47 ++
 .../hdfs/server/namenode/FSDirAppendOp.java     |   3 +-
 .../hdfs/server/namenode/FSDirSnapshotOp.java   |   5 +-
 .../server/namenode/FSDirStatAndListingOp.java  |  41 +-
 .../hdfs/server/namenode/FSDirWriteFileOp.java  |  23 +-
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |   3 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   2 +-
 .../hdfs/server/namenode/FSEditLogOp.java       |  26 +
 .../hdfs/server/namenode/FSNamesystem.java      |  16 +-
 .../hadoop/hdfs/server/namenode/INodeFile.java  |  14 +-
 .../hdfs/server/namenode/NameNodeRpcServer.java |  24 +-
 .../hdfs/server/namenode/NamenodeFsck.java      |   2 +-
 .../hdfs/tools/DelegationTokenFetcher.java      |   3 +-
 .../hdfs/tools/federation/RouterAdmin.java      |   2 +-
 .../src/main/webapps/router/robots.txt          |   2 +
 .../src/site/markdown/HDFSCommands.md           |   2 +-
 .../src/site/markdown/HDFSRouterFederation.md   |   4 +
 .../java/org/apache/hadoop/fs/TestUnbuffer.java |  20 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |  23 +
 .../TestDistributedFileSystemWithECFile.java    |  55 ++
 .../fsdataset/impl/LazyPersistTestCase.java     |   1 -
 .../server/federation/FederationTestUtils.java  |  60 +-
 .../server/federation/RouterDFSCluster.java     |   5 +-
 .../resolver/TestMountTableResolver.java        |  34 +-
 .../federation/router/TestRouterAdmin.java      |  31 +
 .../federation/router/TestRouterMountTable.java | 143 ++++
 .../hdfs/server/namenode/NameNodeAdapter.java   |   7 +-
 .../namenode/OfflineEditsViewerHelper.java      |   4 +-
 .../hdfs/server/namenode/TestBackupNode.java    |   8 +-
 .../hdfs/server/namenode/TestEditLog.java       |   3 +-
 .../hadoop/hdfs/server/namenode/TestFsck.java   |   2 +-
 .../server/namenode/TestNamenodeRetryCache.java |  14 +-
 .../hdfs/server/namenode/TestReencryption.java  |   5 +-
 .../hdfs/server/namenode/TestSaveNamespace.java |   2 +-
 .../server/namenode/ha/TestEditLogTailer.java   |   4 +-
 .../namenode/ha/TestEditLogsDuringFailover.java |  15 +-
 .../namenode/ha/TestFailureToReadEdits.java     |  12 +-
 .../namenode/ha/TestInitializeSharedEdits.java  |   2 +-
 .../namenode/ha/TestRetryCacheWithHA.java       |  13 +-
 .../namenode/ha/TestStandbyInProgressTail.java  |  69 +-
 .../src/test/resources/contract/hdfs.xml        |   2 +-
 .../hadoop-hdfs/src/test/resources/editsStored  | Bin 6753 -> 7909 bytes
 .../src/test/resources/editsStored.xml          | 536 +++++++++----
 .../org/apache/hadoop/mapred/ClientCache.java   |   6 +-
 .../hadoop/mapred/ClientServiceDelegate.java    |   7 +-
 .../hadoop/mapred/ResourceMgrDelegate.java      |   7 +-
 .../org/apache/hadoop/mapred/YARNRunner.java    |   6 +-
 .../org/apache/hadoop/fi/ProbabilityModel.java  |   7 +-
 .../apache/hadoop/fs/AccumulatingReducer.java   |   8 +-
 .../java/org/apache/hadoop/fs/DFSCIOTest.java   |   6 +-
 .../apache/hadoop/fs/DistributedFSCheck.java    |   7 +-
 .../org/apache/hadoop/fs/JHLogAnalyzer.java     |   7 +-
 .../java/org/apache/hadoop/fs/TestDFSIO.java    |   6 +-
 .../java/org/apache/hadoop/fs/TestJHLA.java     |   7 +-
 .../fs/loadGenerator/LoadGeneratorMR.java       |   6 +-
 .../org/apache/hadoop/fs/slive/AppendOp.java    |   6 +-
 .../apache/hadoop/fs/slive/ConfigExtractor.java |   7 +-
 .../org/apache/hadoop/fs/slive/CreateOp.java    |   6 +-
 .../org/apache/hadoop/fs/slive/DeleteOp.java    |   6 +-
 .../java/org/apache/hadoop/fs/slive/ListOp.java |   6 +-
 .../org/apache/hadoop/fs/slive/MkdirOp.java     |   6 +-
 .../java/org/apache/hadoop/fs/slive/ReadOp.java |   6 +-
 .../org/apache/hadoop/fs/slive/RenameOp.java    |   6 +-
 .../apache/hadoop/fs/slive/ReportWriter.java    |   6 +-
 .../org/apache/hadoop/fs/slive/SleepOp.java     |   6 +-
 .../org/apache/hadoop/fs/slive/SliveMapper.java |   6 +-
 .../apache/hadoop/fs/slive/SliveReducer.java    |   6 +-
 .../org/apache/hadoop/fs/slive/SliveTest.java   |   6 +-
 .../org/apache/hadoop/fs/slive/TestSlive.java   |  10 +-
 .../org/apache/hadoop/fs/slive/TruncateOp.java  |   6 +-
 .../apache/hadoop/fs/slive/WeightSelector.java  |   7 +-
 .../java/org/apache/hadoop/hdfs/NNBench.java    |   7 +-
 .../apache/hadoop/hdfs/NNBenchWithoutMR.java    |   8 +-
 .../org/apache/hadoop/mapred/BigMapOutput.java  |   8 +-
 .../java/org/apache/hadoop/mapred/MRBench.java  |   6 +-
 .../org/apache/hadoop/mapred/MiniMRCluster.java |  15 +-
 .../hadoop/mapred/MiniMRYarnClusterAdapter.java |   7 +-
 .../apache/hadoop/mapred/ReliabilityTest.java   |  13 +-
 .../apache/hadoop/mapred/TestBadRecords.java    |   8 +-
 .../hadoop/mapred/TestClientRedirect.java       |   7 +-
 .../mapred/TestCombineFileInputFormat.java      |  11 +-
 .../TestCombineSequenceFileInputFormat.java     |   8 +-
 .../mapred/TestCombineTextInputFormat.java      |   8 +-
 .../mapred/TestConcatenatedCompressedInput.java |   9 +-
 .../mapred/TestFixedLengthInputFormat.java      |  11 +-
 .../apache/hadoop/mapred/TestJobCleanup.java    |   9 +-
 .../hadoop/mapred/TestJobSysDirWithDFS.java     |   8 +-
 .../mapred/TestKeyValueTextInputFormat.java     |   8 +-
 .../mapred/TestMRTimelineEventHandling.java     |   8 +-
 .../apache/hadoop/mapred/TestMapProgress.java   |   8 +-
 .../hadoop/mapred/TestMiniMRChildTask.java      |   8 +-
 .../hadoop/mapred/TestMultiFileInputFormat.java |   8 +-
 .../TestSequenceFileAsBinaryOutputFormat.java   |  13 +-
 .../apache/hadoop/mapred/TestSortedRanges.java  |  12 +-
 .../TestSpecialCharactersInOutputPath.java      |  15 +-
 .../apache/hadoop/mapred/TestTaskStatus.java    |   4 +-
 .../hadoop/mapred/TestTextInputFormat.java      |  11 +-
 .../apache/hadoop/mapred/TestYARNRunner.java    |  14 +-
 .../hadoop/mapred/ThreadedMapBenchmark.java     |   7 +-
 .../org/apache/hadoop/mapred/UtilsForTests.java |   6 +-
 .../mapred/jobcontrol/TestLocalJobControl.java  |   9 +-
 .../apache/hadoop/mapred/pipes/TestPipes.java   |   8 +-
 .../hadoop/mapreduce/GrowingSleepJob.java       |  12 +-
 .../hadoop/mapreduce/JobHistoryFileParser.java  |   7 +-
 .../mapreduce/JobHistoryFileReplayHelper.java   |   9 +-
 .../mapreduce/JobHistoryFileReplayMapperV1.java |   9 +-
 .../mapreduce/JobHistoryFileReplayMapperV2.java |   8 +-
 .../hadoop/mapreduce/MapReduceTestUtil.java     |  13 +-
 .../mapreduce/MiniHadoopClusterManager.java     |   8 +-
 .../hadoop/mapreduce/SimpleEntityWriterV1.java  |   7 +-
 .../hadoop/mapreduce/SimpleEntityWriterV2.java  |   9 +-
 .../apache/hadoop/mapreduce/TestCounters.java   |  11 +-
 .../hadoop/mapreduce/TestLocalRunner.java       |  11 +-
 .../hadoop/mapreduce/TestMRJobClient.java       |   7 +-
 .../hadoop/mapreduce/TestMapCollection.java     |  13 +-
 .../hadoop/mapreduce/TestValueIterReset.java    |   8 +-
 .../mapreduce/TimelineEntityConverterV1.java    |   8 +-
 .../mapreduce/TimelineEntityConverterV2.java    |   8 +-
 .../lib/db/TestDataDrivenDBInputFormat.java     |   6 +-
 .../TestCombineSequenceFileInputFormat.java     |   8 +-
 .../lib/input/TestCombineTextInputFormat.java   |   8 +-
 .../lib/input/TestFixedLengthInputFormat.java   |  12 +-
 .../input/TestMRKeyValueTextInputFormat.java    |  12 +-
 .../input/TestMRSequenceFileInputFilter.java    |   8 +-
 .../lib/jobcontrol/TestMapReduceJobControl.java |  12 +-
 .../TestMRSequenceFileAsBinaryOutputFormat.java |   8 +-
 .../lib/partition/TestKeyFieldHelper.java       |  10 +-
 .../mapreduce/security/TestJHSSecurity.java     |  10 +-
 .../mapreduce/util/MRAsyncDiskService.java      |   7 +-
 .../mapreduce/util/TestMRAsyncDiskService.java  |   9 +-
 .../hadoop/mapreduce/v2/MiniMRYarnCluster.java  |   7 +-
 .../TestMRAMWithNonNormalizedCapabilities.java  |   7 +-
 .../mapreduce/v2/TestMRAppWithCombiner.java     |   7 +-
 .../apache/hadoop/mapreduce/v2/TestMRJobs.java  |   6 +-
 .../v2/TestMRJobsWithHistoryService.java        |   8 +-
 .../mapreduce/v2/TestMRJobsWithProfiler.java    |   8 +-
 .../hadoop/mapreduce/v2/TestMROldApiJobs.java   |   7 +-
 .../hadoop/mapreduce/v2/TestRMNMInfo.java       |   7 +-
 .../mapreduce/v2/TestSpeculativeExecution.java  |   7 +-
 .../apache/hadoop/mapreduce/v2/TestUberAM.java  |   6 +-
 .../test/java/testjar/UserNamePermission.java   |  13 +-
 .../mapred/nativetask/HadoopPlatform.java       |   7 +-
 .../mapred/nativetask/NativeBatchProcessor.java |   9 +-
 .../NativeMapOutputCollectorDelegator.java      |   7 +-
 .../hadoop/mapred/nativetask/NativeRuntime.java |   7 +-
 .../hadoop/mapred/nativetask/Platforms.java     |   6 +-
 .../mapred/nativetask/StatusReportChecker.java  |   7 +-
 .../nativetask/handlers/BufferPuller.java       |   6 +-
 .../nativetask/handlers/BufferPushee.java       |   6 +-
 .../nativetask/handlers/BufferPusher.java       |   6 +-
 .../nativetask/handlers/CombinerHandler.java    |   7 +-
 .../handlers/NativeCollectorOnlyHandler.java    |   7 +-
 .../mapred/nativetask/serde/KVSerializer.java   |   7 +-
 .../mapred/nativetask/util/OutputUtil.java      |   6 +-
 .../combinertest/LargeKVCombinerTest.java       |   7 +-
 .../nativetask/combinertest/WordCount.java      |   6 +-
 .../hadoop/mapred/nativetask/kvtest/KVJob.java  |   6 +-
 .../hadoop/mapred/nativetask/kvtest/KVTest.java |   7 +-
 .../mapred/nativetask/kvtest/LargeKVTest.java   |   6 +-
 .../mapred/nativetask/kvtest/TestInputFile.java |   7 +-
 .../EnforceNativeOutputCollectorDelegator.java  |   7 +-
 .../mapred/uploader/FrameworkUploader.java      |  59 +-
 .../mapred/uploader/TestFrameworkUploader.java  |  79 +-
 hadoop-project/pom.xml                          |   2 +-
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |   2 +-
 .../apache/hadoop/tools/DistCpOptionSwitch.java |   2 +-
 .../src/site/markdown/DistCp.md.vm              |   2 +-
 .../hadoop-yarn/conf/container-executor.cfg     |   6 +
 .../yarn/api/records/QueueConfigurations.java   |  76 ++
 .../yarn/api/records/ResourceInformation.java   |   6 +
 .../src/main/proto/yarn_protos.proto            |   4 +
 .../hadoop/yarn/service/ServiceScheduler.java   |  72 +-
 .../yarn/service/component/Component.java       |   8 +
 .../component/instance/ComponentInstance.java   |  10 +-
 .../yarn/service/conf/YarnServiceConf.java      |  10 +
 .../hadoop/yarn/service/MockServiceAM.java      | 144 +++-
 .../hadoop/yarn/service/TestServiceAM.java      |  98 ++-
 .../yarn/client/api/async/AMRMClientAsync.java  |  10 +
 .../api/async/impl/AMRMClientAsyncImpl.java     |   5 +
 .../impl/pb/QueueConfigurationsPBImpl.java      | 136 +++-
 .../api/records/impl/pb/ResourcePBImpl.java     |   1 -
 .../resource/DefaultResourceCalculator.java     |  14 +
 .../resource/DominantResourceCalculator.java    |  46 ++
 .../yarn/util/resource/ResourceCalculator.java  |  32 +-
 .../hadoop/yarn/util/resource/Resources.java    |  10 +
 .../types/yarn/YarnRegistryAttributes.java      |   1 +
 .../hadoop/yarn/server/webapp/WebPageUtils.java |   4 +-
 .../yarn/server/MockResourceManagerFacade.java  |  16 +-
 .../src/CMakeLists.txt                          |   2 +
 .../hadoop/yarn/server/nodemanager/Context.java |   3 +
 .../nodemanager/DefaultContainerExecutor.java   |  25 +-
 .../yarn/server/nodemanager/NodeManager.java    |  19 +-
 .../nodemanager/NodeResourceMonitorImpl.java    |  16 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |   5 +-
 .../amrmproxy/FederationInterceptor.java        | 271 ++++++-
 .../container/ContainerImpl.java                |  20 +-
 .../launcher/ContainerLaunch.java               |  54 +-
 .../JavaSandboxLinuxContainerRuntime.java       |  17 +-
 .../monitor/ContainersMonitorImpl.java          |  13 +
 .../nodemanager/metrics/NodeManagerMetrics.java |  61 ++
 .../main/native/container-executor/impl/main.c  |   6 +
 .../impl/modules/fpga/fpga-module.c             | 229 ++++++
 .../impl/modules/fpga/fpga-module.h             |  45 ++
 .../test/modules/fpga/test-fpga-module.cc       | 216 ++++++
 .../nodemanager/TestNodeResourceMonitor.java    |  34 +-
 .../amrmproxy/BaseAMRMProxyTest.java            |  21 +
 .../amrmproxy/TestFederationInterceptor.java    | 104 +++
 .../TestableFederationInterceptor.java          |   8 +-
 .../monitor/MockResourceCalculatorPlugin.java   |   5 +
 .../monitor/TestContainersMonitor.java          |  20 +-
 .../server/resourcemanager/RMAppManager.java    |  10 +-
 .../monitor/SchedulingMonitor.java              |   4 +-
 .../AbstractPreemptableResourceCalculator.java  |  24 +-
 .../capacity/FifoCandidatesSelector.java        |   9 +-
 .../ProportionalCapacityPreemptionPolicy.java   |  17 +-
 .../monitor/capacity/TempQueuePerPartition.java |  36 +-
 .../CapacitySchedulerPlanFollower.java          |  12 +-
 .../scheduler/AbstractResourceUsage.java        | 210 ++++++
 .../scheduler/AbstractYarnScheduler.java        |   2 +-
 .../scheduler/QueueResourceQuotas.java          | 115 +++
 .../scheduler/ResourceUsage.java                | 243 ++----
 .../scheduler/YarnScheduler.java                |   2 +-
 .../capacity/AbstractAutoCreatedLeafQueue.java  | 113 +++
 .../scheduler/capacity/AbstractCSQueue.java     | 265 ++++++-
 .../capacity/AbstractManagedParentQueue.java    | 162 ++--
 .../capacity/AutoCreatedLeafQueue.java          | 147 ++--
 .../capacity/AutoCreatedLeafQueueConfig.java    |  66 ++
 .../AutoCreatedQueueManagementPolicy.java       |  64 ++
 .../scheduler/capacity/CSQueue.java             |  65 +-
 .../scheduler/capacity/CSQueueUtils.java        |  65 +-
 .../scheduler/capacity/CapacityScheduler.java   | 108 +--
 .../CapacitySchedulerConfiguration.java         | 358 ++++++++-
 .../capacity/CapacitySchedulerContext.java      |   8 +
 .../capacity/CapacitySchedulerQueueManager.java |   6 +-
 .../scheduler/capacity/LeafQueue.java           | 127 +++-
 .../scheduler/capacity/ManagedParentQueue.java  | 294 +++++++-
 .../scheduler/capacity/ParentQueue.java         | 305 +++++++-
 .../scheduler/capacity/PlanQueue.java           | 150 +++-
 .../capacity/QueueManagementChange.java         | 148 ++++
 .../QueueManagementDynamicEditPolicy.java       | 272 +++++++
 .../scheduler/capacity/ReservationQueue.java    |  91 +++
 .../scheduler/capacity/UsersManager.java        |   5 +-
 .../PriorityUtilizationQueueOrderingPolicy.java |  82 +-
 .../GuaranteedOrZeroCapacityOverTimePolicy.java | 745 +++++++++++++++++++
 .../scheduler/common/QueueEntitlement.java      |  22 +
 .../scheduler/common/fica/FiCaSchedulerApp.java |  28 +-
 .../event/QueueManagementChangeEvent.java       |  49 ++
 .../scheduler/event/SchedulerEventType.java     |   5 +-
 .../webapp/CapacitySchedulerPage.java           |  64 +-
 .../webapp/dao/CapacitySchedulerInfo.java       |   3 +-
 .../dao/CapacitySchedulerLeafQueueInfo.java     |  17 +-
 .../webapp/dao/CapacitySchedulerQueueInfo.java  |  29 +-
 .../dao/PartitionQueueCapacitiesInfo.java       |  34 +-
 .../webapp/dao/QueueCapacitiesInfo.java         |  13 +-
 .../yarn/server/resourcemanager/MockNM.java     |   8 +
 .../yarn/server/resourcemanager/MockRM.java     |   6 +
 .../server/resourcemanager/TestAppManager.java  | 112 ++-
 ...alCapacityPreemptionPolicyMockFramework.java |  27 +
 ...estProportionalCapacityPreemptionPolicy.java |  28 +-
 ...pacityPreemptionPolicyForNodePartitions.java |  76 ++
 ...pacityPreemptionPolicyIntraQueueWithDRF.java |   6 +-
 .../TestAbsoluteResourceConfiguration.java      | 630 ++++++++++++++++
 .../capacity/TestApplicationLimits.java         |  30 +-
 .../TestApplicationLimitsByPartition.java       |   4 +
 .../capacity/TestAutoCreatedLeafQueue.java      | 113 ---
 ...stCapacitySchedulerAutoCreatedQueueBase.java | 683 +++++++++++++++++
 .../TestCapacitySchedulerAutoQueueCreation.java | 703 ++++++++---------
 .../TestCapacitySchedulerDynamicBehavior.java   |  32 +-
 ...CapacitySchedulerWithMultiResourceTypes.java |  25 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   2 +
 ...tGuaranteedOrZeroCapacityOverTimePolicy.java |  40 +
 .../scheduler/capacity/TestLeafQueue.java       | 492 +++++++-----
 .../TestNodeLabelContainerAllocation.java       | 163 ++++
 .../scheduler/capacity/TestParentQueue.java     | 102 +++
 .../TestQueueManagementDynamicEditPolicy.java   | 121 +++
 .../scheduler/capacity/TestQueueParsing.java    | 168 ++++-
 .../capacity/TestReservationQueue.java          | 114 +++
 .../scheduler/capacity/TestReservations.java    |  18 +
 .../scheduler/capacity/TestUtils.java           |   2 +-
 ...tPriorityUtilizationQueueOrderingPolicy.java |  81 +-
 .../webapp/TestRMWebServicesCapacitySched.java  |   4 +-
 .../src/site/markdown/CapacityScheduler.md      |  10 +-
 .../markdown/yarn-service/Configurations.md     |   1 +
 302 files changed, 10651 insertions(+), 2587 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/hadoop/blob/070ad843/hadoop-project/pom.xml
----------------------------------------------------------------------


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[38/50] hadoop git commit: YARN-7608. Incorrect sTarget column causing DataTable warning on RM application and scheduler web page. Contributed by Gergely Novák.

Posted by xy...@apache.org.
YARN-7608. Incorrect sTarget column causing DataTable warning on RM application and scheduler web page. Contributed by Gergely Novák.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2edc4cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2edc4cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2edc4cb

Branch: refs/heads/HDFS-7240
Commit: a2edc4cbf5c1f7bc38315c52391362fbbc48fab1
Parents: 670e8d4
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Dec 11 10:31:46 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Dec 11 10:31:46 2017 +0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2edc4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 3a26ae5..29f1e63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -55,9 +55,9 @@ public class WebPageUtils {
       .append(", 'mRender': renderHadoopDate }")
       .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':");
     if (isFairSchedulerPage) {
-      sb.append("[13]");
-    } else if (isResourceManager) {
       sb.append("[15]");
+    } else if (isResourceManager) {
+      sb.append("[17]");
     } else {
       sb.append("[9]");
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[21/50] hadoop git commit: YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index e3c108a..b9d5b82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableTable;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.junit.Assert;
@@ -52,6 +53,8 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
       when(q.getQueueCapacities()).thenReturn(qc);
       when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i]));
 
+      QueueResourceQuotas qr = new QueueResourceQuotas();
+      when(q.getQueueResourceQuotas()).thenReturn(qr);
       list.add(q);
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e798b1a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1108f1a..0132348 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -354,10 +354,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
   private void verifySubQueue(JSONObject info, String q, 
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
-    int numExpectedElements = 18;
+    int numExpectedElements = 20;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 31;
+      numExpectedElements = 33;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[47/50] hadoop git commit: YARN-7625. Expose NM node/containers resource utilization in JVM metrics. Contributed by Weiwei Yang

Posted by xy...@apache.org.
YARN-7625. Expose NM node/containers resource utilization in JVM metrics. Contributed by Weiwei Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06f0eb2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06f0eb2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06f0eb2d

Branch: refs/heads/HDFS-7240
Commit: 06f0eb2dce2a7a098f7844682ea6c232d0ddb0be
Parents: 8bb83a8
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Dec 12 12:56:26 2017 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Dec 12 12:56:26 2017 -0600

----------------------------------------------------------------------
 .../hadoop/yarn/server/nodemanager/Context.java |  3 +
 .../yarn/server/nodemanager/NodeManager.java    | 19 +++++-
 .../nodemanager/NodeResourceMonitorImpl.java    | 16 ++++-
 .../monitor/ContainersMonitorImpl.java          | 13 +++++
 .../nodemanager/metrics/NodeManagerMetrics.java | 61 ++++++++++++++++++++
 .../nodemanager/TestNodeResourceMonitor.java    | 34 ++++++++++-
 .../amrmproxy/BaseAMRMProxyTest.java            |  6 ++
 .../monitor/MockResourceCalculatorPlugin.java   |  5 ++
 .../monitor/TestContainersMonitor.java          | 20 ++++++-
 9 files changed, 170 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index a1c474f..6774cf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
@@ -125,4 +126,6 @@ public interface Context {
   ContainerStateTransitionListener getContainerStateTransitionListener();
 
   ResourcePluginManager getResourcePluginManager();
+
+  NodeManagerMetrics getNodeManagerMetrics();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index bddc7c3..179b01e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -205,7 +205,7 @@ public class NodeManager extends CompositeService
   }
 
   protected NodeResourceMonitor createNodeResourceMonitor() {
-    return new NodeResourceMonitorImpl();
+    return new NodeResourceMonitorImpl(context);
   }
 
   protected ContainerManagerImpl createContainerManager(Context context,
@@ -242,6 +242,7 @@ public class NodeManager extends CompositeService
     NMContext nmContext = new NMContext(containerTokenSecretManager,
         nmTokenSecretManager, dirsHandler, aclsManager, stateStore,
         isDistSchedulerEnabled, conf);
+    nmContext.setNodeManagerMetrics(metrics);
     DefaultContainerStateListener defaultListener =
         new DefaultContainerStateListener();
     nmContext.setContainerStateTransitionListener(defaultListener);
@@ -574,6 +575,8 @@ public class NodeManager extends CompositeService
 
     private Configuration conf = null;
 
+    private NodeManagerMetrics metrics = null;
+
     protected final ConcurrentMap<ApplicationId, Application> applications =
         new ConcurrentHashMap<ApplicationId, Application>();
 
@@ -823,6 +826,20 @@ public class NodeManager extends CompositeService
       return resourcePluginManager;
     }
 
+    /**
+     * Returns the {@link NodeManagerMetrics} instance of this node.
+     * This might return a null if the instance was not set to the context.
+     * @return node manager metrics.
+     */
+    @Override
+    public NodeManagerMetrics getNodeManagerMetrics() {
+      return metrics;
+    }
+
+    public void setNodeManagerMetrics(NodeManagerMetrics nmMetrics) {
+      this.metrics = nmMetrics;
+    }
+
     public void setResourcePluginManager(
         ResourcePluginManager resourcePluginManager) {
       this.resourcePluginManager = resourcePluginManager;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index 8b96ba5..d489835 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,12 +49,14 @@ public class NodeResourceMonitorImpl extends AbstractService implements
   /** Current <em>resource utilization</em> of the node. */
   private ResourceUtilization nodeUtilization;
 
+  private Context nmContext;
+
   /**
    * Initialize the node resource monitor.
    */
-  public NodeResourceMonitorImpl() {
+  public NodeResourceMonitorImpl(Context context) {
     super(NodeResourceMonitorImpl.class.getName());
-
+    this.nmContext = context;
     this.monitoringThread = new MonitoringThread();
   }
 
@@ -149,6 +152,15 @@ public class NodeResourceMonitorImpl extends AbstractService implements
                 (int) (vmem >> 20), // B -> MB
                 vcores); // Used Virtual Cores
 
+        // Publish the node utilization metrics to node manager
+        // metrics system.
+        NodeManagerMetrics nmMetrics = nmContext.getNodeManagerMetrics();
+        if (nmMetrics != null) {
+          nmMetrics.setNodeUsedMemGB(nodeUtilization.getPhysicalMemory());
+          nmMetrics.setNodeUsedVMemGB(nodeUtilization.getVirtualMemory());
+          nmMetrics.setNodeCpuUtilization(nodeUtilization.getCPU());
+        }
+
         try {
           Thread.sleep(monitoringInterval);
         } catch (InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 2b99cc7..48ec147 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -488,6 +489,18 @@ public class ContainersMonitorImpl extends AbstractService implements
         // Save the aggregated utilization of the containers
         setContainersUtilization(trackedContainersUtilization);
 
+        // Publish the container utilization metrics to node manager
+        // metrics system.
+        NodeManagerMetrics nmMetrics = context.getNodeManagerMetrics();
+        if (nmMetrics != null) {
+          nmMetrics.setContainerUsedMemGB(
+              trackedContainersUtilization.getPhysicalMemory());
+          nmMetrics.setContainerUsedVMemGB(
+              trackedContainersUtilization.getVirtualMemory());
+          nmMetrics.setContainerCpuUtilization(
+              trackedContainersUtilization.getCPU());
+        }
+
         try {
           Thread.sleep(monitoringInterval);
         } catch (InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
index a59bb5c..f0abfd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableCounterInt;
 import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
 import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
 import org.apache.hadoop.metrics2.lib.MutableRate;
 import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -77,6 +78,18 @@ public class NodeManagerMetrics {
   MutableGaugeLong publicBytesDeleted;
   @Metric("# of bytes deleted from the private local cache")
   MutableGaugeLong privateBytesDeleted;
+  @Metric("Current used physical memory by all containers in GB")
+  MutableGaugeInt containerUsedMemGB;
+  @Metric("Current used virtual memory by all containers in GB")
+  MutableGaugeInt containerUsedVMemGB;
+  @Metric("Aggregated CPU utilization of all containers")
+  MutableGaugeFloat containerCpuUtilization;
+  @Metric("Current used memory by this node in GB")
+  MutableGaugeInt nodeUsedMemGB;
+  @Metric("Current used virtual memory by this node in GB")
+  MutableGaugeInt nodeUsedVMemGB;
+  @Metric("Current CPU utilization")
+  MutableGaugeFloat nodeCpuUtilization;
 
   // CHECKSTYLE:ON:VisibilityModifier
 
@@ -316,4 +329,52 @@ public class NodeManagerMetrics {
   public long getPrivateBytesDeleted() {
     return this.privateBytesDeleted.value();
   }
+
+  public void setContainerUsedMemGB(long usedMem) {
+    this.containerUsedMemGB.set((int)Math.floor(usedMem/1024d));
+  }
+
+  public int getContainerUsedMemGB() {
+    return this.containerUsedMemGB.value();
+  }
+
+  public void setContainerUsedVMemGB(long usedVMem) {
+    this.containerUsedVMemGB.set((int)Math.floor(usedVMem/1024d));
+  }
+
+  public int getContainerUsedVMemGB() {
+    return this.containerUsedVMemGB.value();
+  }
+
+  public void setContainerCpuUtilization(float cpuUtilization) {
+    this.containerCpuUtilization.set(cpuUtilization);
+  }
+
+  public float getContainerCpuUtilization() {
+    return this.containerCpuUtilization.value();
+  }
+
+  public void setNodeUsedMemGB(long totalUsedMemGB) {
+    this.nodeUsedMemGB.set((int)Math.floor(totalUsedMemGB/1024d));
+  }
+
+  public int getNodeUsedMemGB() {
+    return nodeUsedMemGB.value();
+  }
+
+  public void setNodeUsedVMemGB(long totalUsedVMemGB) {
+    this.nodeUsedVMemGB.set((int)Math.floor(totalUsedVMemGB/1024d));
+  }
+
+  public int getNodeUsedVMemGB() {
+    return nodeUsedVMemGB.value();
+  }
+
+  public float getNodeCpuUtilization() {
+    return nodeCpuUtilization.value();
+  }
+
+  public void setNodeCpuUtilization(float cpuUtilization) {
+    this.nodeCpuUtilization.set(cpuUtilization);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
index 3c2c386..000e8ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
@@ -18,18 +18,46 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import java.io.IOException;
+
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager
+    .BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager
+    .monitor.MockResourceCalculatorPlugin;
 
+import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
 
 public class TestNodeResourceMonitor extends BaseContainerManagerTest {
   public TestNodeResourceMonitor() throws UnsupportedFileSystemException {
     super();
   }
 
+  @Before
+  public void setup() throws IOException {
+    // Enable node resource monitor with a mocked resource calculator.
+    conf.set(
+        YarnConfiguration.NM_MON_RESOURCE_CALCULATOR,
+        MockResourceCalculatorPlugin.class.getCanonicalName());
+    super.setup();
+  }
+
   @Test
-  public void testNodeResourceMonitor() {
-    NodeResourceMonitor nrm = new NodeResourceMonitorImpl();
+  public void testMetricsUpdate() throws Exception {
+    // This test doesn't verify the correction of those metrics
+    // updated by the monitor, it only verifies that the monitor
+    // do publish these info to node manager metrics system in
+    // each monitor interval.
+    Context spyContext = spy(context);
+    NodeResourceMonitor nrm = new NodeResourceMonitorImpl(spyContext);
+    nrm.init(conf);
+    nrm.start();
+    Mockito.verify(spyContext, timeout(500).atLeastOnce())
+        .getNodeManagerMetrics();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 0319dbe..18d1dce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManag
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
@@ -802,5 +803,10 @@ public abstract class BaseAMRMProxyTest {
     public ResourcePluginManager getResourcePluginManager() {
       return null;
     }
+
+    @Override
+    public NodeManagerMetrics getNodeManagerMetrics() {
+      return null;
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
index 0dc5c5b..64d117a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorPlugin.java
@@ -70,4 +70,9 @@ public class MockResourceCalculatorPlugin extends ResourceCalculatorPlugin {
   public float getCpuUsagePercentage() {
     return 0;
   }
+
+  @Override
+  public float getNumVCoresUsed() {
+    return 0;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f0eb2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index 6f7fadf..5f72a4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -22,6 +22,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -67,7 +69,6 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
 import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
@@ -75,6 +76,7 @@ import org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
 import org.slf4j.LoggerFactory;
 
 public class TestContainersMonitor extends BaseContainerManagerTest {
@@ -95,6 +97,22 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
     super.setup();
   }
 
+  @Test
+  public void testMetricsUpdate() throws Exception {
+    // This test doesn't verify the correction of those metrics
+    // updated by the monitor, it only verifies that the monitor
+    // do publish these info to node manager metrics system in
+    // each monitor interval.
+    Context spyContext = spy(context);
+    ContainersMonitorImpl cm =
+        new ContainersMonitorImpl(mock(ContainerExecutor.class),
+            mock(AsyncDispatcher.class), spyContext);
+    cm.init(getConfForCM(false, true, 1024, 2.1f));
+    cm.start();
+    Mockito.verify(spyContext, timeout(500).atLeastOnce())
+        .getNodeManagerMetrics();
+  }
+
   /**
    * Test to verify the check for whether a process tree is over limit or not.
    *


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[37/50] hadoop git commit: YARN-6704. Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService. (Botong Huang via Subru).

Posted by xy...@apache.org.
YARN-6704. Add support for work preserving NM restart when FederationInterceptor is enabled in AMRMProxyService. (Botong Huang via Subru).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/670e8d4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/670e8d4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/670e8d4e

Branch: refs/heads/HDFS-7240
Commit: 670e8d4ec7e71fc3b054cd3b2826f869b649a788
Parents: 04b84da
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Dec 8 15:39:18 2017 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Dec 8 15:39:18 2017 -0800

----------------------------------------------------------------------
 .../yarn/server/MockResourceManagerFacade.java  |  16 +-
 .../nodemanager/amrmproxy/AMRMProxyService.java |   5 +-
 .../amrmproxy/FederationInterceptor.java        | 271 +++++++++++++++++--
 .../amrmproxy/BaseAMRMProxyTest.java            |  15 +
 .../amrmproxy/TestFederationInterceptor.java    | 104 +++++++
 .../TestableFederationInterceptor.java          |   8 +-
 6 files changed, 387 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index b5727aa..15e1cea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -111,6 +111,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -622,7 +623,20 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
 
     validateRunning();
 
-    return GetContainersResponse.newInstance(null);
+    ApplicationAttemptId attemptId = request.getApplicationAttemptId();
+    List<ContainerReport> containers = new ArrayList<>();
+    synchronized (applicationContainerIdMap) {
+      // Return the list of running containers that were being tracked for this
+      // application
+      Assert.assertTrue("The application id is NOT registered: " + attemptId,
+          applicationContainerIdMap.containsKey(attemptId));
+      List<ContainerId> ids = applicationContainerIdMap.get(attemptId);
+      for (ContainerId c : ids) {
+        containers.add(ContainerReport.newInstance(c, null, null, null, 0, 0,
+            null, null, 0, null, null));
+      }
+    }
+    return GetContainersResponse.newInstance(containers);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index ebd85bf..815e39b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -128,11 +128,8 @@ public class AMRMProxyService extends CompositeService implements
         new AMRMProxyTokenSecretManager(this.nmContext.getNMStateStore());
     this.secretManager.init(conf);
 
-    // Both second app attempt and NM restart within Federation need registry
     if (conf.getBoolean(YarnConfiguration.AMRM_PROXY_HA_ENABLED,
-        YarnConfiguration.DEFAULT_AMRM_PROXY_HA_ENABLED)
-        || conf.getBoolean(YarnConfiguration.NM_RECOVERY_ENABLED,
-            YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED)) {
+        YarnConfiguration.DEFAULT_AMRM_PROXY_HA_ENABLED)) {
       this.registry = FederationStateStoreFacade.createInstance(conf,
           YarnConfiguration.YARN_REGISTRY_CLASS,
           YarnConfiguration.DEFAULT_YARN_REGISTRY_CLASS,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index ef5e061..9a53a50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -37,16 +37,23 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.RegisterApplicationMasterResponsePBImpl;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.PreemptionContract;
@@ -59,6 +66,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
@@ -90,6 +99,22 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   private static final Logger LOG =
       LoggerFactory.getLogger(FederationInterceptor.class);
 
+  public static final String NMSS_CLASS_PREFIX = "FederationInterceptor/";
+
+  public static final String NMSS_REG_REQUEST_KEY =
+      NMSS_CLASS_PREFIX + "registerRequest";
+  public static final String NMSS_REG_RESPONSE_KEY =
+      NMSS_CLASS_PREFIX + "registerResponse";
+
+  /*
+   * When AMRMProxy HA is enabled, secondary AMRMTokens will be stored in Yarn
+   * Registry. Otherwise if NM recovery is enabled, the UAM token are store in
+   * local NMSS instead under this directory name.
+   */
+  public static final String NMSS_SECONDARY_SC_PREFIX =
+      NMSS_CLASS_PREFIX + "secondarySC/";
+  public static final String STRING_TO_BYTE_FORMAT = "UTF-8";
+
   /**
    * The home sub-cluster is the sub-cluster where the AM container is running
    * in.
@@ -187,14 +212,20 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     } catch (Exception ex) {
       throw new YarnRuntimeException(ex);
     }
-    // Add all app tokens for Yarn Registry access
-    if (this.registryClient != null && appContext.getCredentials() != null) {
-      this.appOwner.addCredentials(appContext.getCredentials());
+
+    if (appContext.getRegistryClient() != null) {
+      this.registryClient = new FederationRegistryClient(conf,
+          appContext.getRegistryClient(), this.appOwner);
+      // Add all app tokens for Yarn Registry access
+      if (appContext.getCredentials() != null) {
+        this.appOwner.addCredentials(appContext.getCredentials());
+      }
     }
 
     this.homeSubClusterId =
         SubClusterId.newInstance(YarnConfiguration.getClusterId(conf));
-    this.homeRM = createHomeRMProxy(appContext);
+    this.homeRM = createHomeRMProxy(appContext, ApplicationMasterProtocol.class,
+        this.appOwner);
 
     this.federationFacade = FederationStateStoreFacade.getInstance();
     this.subClusterResolver = this.federationFacade.getSubClusterResolver();
@@ -204,11 +235,137 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
 
     this.uamPool.init(conf);
     this.uamPool.start();
+  }
 
-    if (appContext.getRegistryClient() != null) {
-      this.registryClient = new FederationRegistryClient(conf,
-          appContext.getRegistryClient(), this.appOwner);
+  @Override
+  public void recover(Map<String, byte[]> recoveredDataMap) {
+    super.recover(recoveredDataMap);
+    LOG.info("Recovering data for FederationInterceptor");
+    if (recoveredDataMap == null) {
+      return;
     }
+
+    ApplicationAttemptId attemptId =
+        getApplicationContext().getApplicationAttemptId();
+    try {
+      if (recoveredDataMap.containsKey(NMSS_REG_REQUEST_KEY)) {
+        RegisterApplicationMasterRequestProto pb =
+            RegisterApplicationMasterRequestProto
+                .parseFrom(recoveredDataMap.get(NMSS_REG_REQUEST_KEY));
+        this.amRegistrationRequest =
+            new RegisterApplicationMasterRequestPBImpl(pb);
+        LOG.info("amRegistrationRequest recovered for {}", attemptId);
+      }
+      if (recoveredDataMap.containsKey(NMSS_REG_RESPONSE_KEY)) {
+        RegisterApplicationMasterResponseProto pb =
+            RegisterApplicationMasterResponseProto
+                .parseFrom(recoveredDataMap.get(NMSS_REG_RESPONSE_KEY));
+        this.amRegistrationResponse =
+            new RegisterApplicationMasterResponsePBImpl(pb);
+        LOG.info("amRegistrationResponse recovered for {}", attemptId);
+      }
+
+      // Recover UAM amrmTokens from registry or NMSS
+      Map<String, Token<AMRMTokenIdentifier>> uamMap;
+      if (this.registryClient != null) {
+        uamMap = this.registryClient
+            .loadStateFromRegistry(attemptId.getApplicationId());
+        LOG.info("Found {} existing UAMs for application {} in Yarn Registry",
+            uamMap.size(), attemptId.getApplicationId());
+      } else {
+        uamMap = new HashMap<>();
+        for (Entry<String, byte[]> entry : recoveredDataMap.entrySet()) {
+          if (entry.getKey().startsWith(NMSS_SECONDARY_SC_PREFIX)) {
+            // entry for subClusterId -> UAM amrmToken
+            String scId =
+                entry.getKey().substring(NMSS_SECONDARY_SC_PREFIX.length());
+            Token<AMRMTokenIdentifier> amrmToken = new Token<>();
+            amrmToken.decodeFromUrlString(
+                new String(entry.getValue(), STRING_TO_BYTE_FORMAT));
+            uamMap.put(scId, amrmToken);
+            LOG.debug("Recovered UAM in " + scId + " from NMSS");
+          }
+        }
+        LOG.info("Found {} existing UAMs for application {} in NMStateStore",
+            uamMap.size(), attemptId.getApplicationId());
+      }
+
+      // Re-attach the UAMs
+      int containers = 0;
+      for (Map.Entry<String, Token<AMRMTokenIdentifier>> entry : uamMap
+          .entrySet()) {
+        SubClusterId subClusterId = SubClusterId.newInstance(entry.getKey());
+
+        // Create a config loaded with federation on and subclusterId
+        // for each UAM
+        YarnConfiguration config = new YarnConfiguration(getConf());
+        FederationProxyProviderUtil.updateConfForFederation(config,
+            subClusterId.getId());
+
+        try {
+          this.uamPool.reAttachUAM(subClusterId.getId(), config,
+              attemptId.getApplicationId(),
+              this.amRegistrationResponse.getQueue(),
+              getApplicationContext().getUser(), this.homeSubClusterId.getId(),
+              entry.getValue());
+
+          RegisterApplicationMasterResponse response =
+              this.uamPool.registerApplicationMaster(subClusterId.getId(),
+                  this.amRegistrationRequest);
+
+          // Running containers from secondary RMs
+          for (Container container : response
+              .getContainersFromPreviousAttempts()) {
+            containerIdToSubClusterIdMap.put(container.getId(), subClusterId);
+            containers++;
+          }
+          LOG.info("Recovered {} running containers from UAM in {}",
+              response.getContainersFromPreviousAttempts().size(),
+              subClusterId);
+
+        } catch (Exception e) {
+          LOG.error(
+              "Error reattaching UAM to " + subClusterId + " for " + attemptId,
+              e);
+        }
+      }
+
+      // Get the running containers from home RM, note that we will also get the
+      // AM container itself from here. We don't need it, but no harm to put the
+      // map as well.
+      UserGroupInformation appSubmitter = UserGroupInformation
+          .createRemoteUser(getApplicationContext().getUser());
+      ApplicationClientProtocol rmClient =
+          createHomeRMProxy(getApplicationContext(),
+              ApplicationClientProtocol.class, appSubmitter);
+
+      GetContainersResponse response =
+          rmClient.getContainers(GetContainersRequest.newInstance(attemptId));
+      for (ContainerReport container : response.getContainerList()) {
+        containerIdToSubClusterIdMap.put(container.getContainerId(),
+            this.homeSubClusterId);
+        containers++;
+        LOG.debug("  From home RM " + this.homeSubClusterId
+            + " running container " + container.getContainerId());
+      }
+      LOG.info("{} running containers including AM recovered from home RM ",
+          response.getContainerList().size(), this.homeSubClusterId);
+
+      LOG.info(
+          "In all {} UAMs {} running containers including AM recovered for {}",
+          uamMap.size(), containers, attemptId);
+
+      if (this.amRegistrationResponse != null) {
+        // Initialize the AMRMProxyPolicy
+        String queue = this.amRegistrationResponse.getQueue();
+        this.policyInterpreter =
+            FederationPolicyUtils.loadAMRMPolicy(queue, this.policyInterpreter,
+                getConf(), this.federationFacade, this.homeSubClusterId);
+      }
+    } catch (IOException | YarnException e) {
+      throw new YarnRuntimeException(e);
+    }
+
   }
 
   /**
@@ -242,6 +399,19 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       // Save the registration request. This will be used for registering with
       // secondary sub-clusters using UAMs, as well as re-register later
       this.amRegistrationRequest = request;
+      if (getNMStateStore() != null) {
+        try {
+          RegisterApplicationMasterRequestPBImpl pb =
+              (RegisterApplicationMasterRequestPBImpl)
+                  this.amRegistrationRequest;
+          getNMStateStore().storeAMRMProxyAppContextEntry(
+              getApplicationContext().getApplicationAttemptId(),
+              NMSS_REG_REQUEST_KEY, pb.getProto().toByteArray());
+        } catch (Exception e) {
+          LOG.error("Error storing AMRMProxy application context entry for "
+              + getApplicationContext().getApplicationAttemptId(), e);
+        }
+      }
     }
 
     /*
@@ -278,6 +448,20 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
         getApplicationContext().getApplicationAttemptId().getApplicationId();
     reAttachUAMAndMergeRegisterResponse(this.amRegistrationResponse, appId);
 
+    if (getNMStateStore() != null) {
+      try {
+        RegisterApplicationMasterResponsePBImpl pb =
+            (RegisterApplicationMasterResponsePBImpl)
+                this.amRegistrationResponse;
+        getNMStateStore().storeAMRMProxyAppContextEntry(
+            getApplicationContext().getApplicationAttemptId(),
+            NMSS_REG_RESPONSE_KEY, pb.getProto().toByteArray());
+      } catch (Exception e) {
+        LOG.error("Error storing AMRMProxy application context entry for "
+            + getApplicationContext().getApplicationAttemptId(), e);
+      }
+    }
+
     // the queue this application belongs will be used for getting
     // AMRMProxy policy from state store.
     String queue = this.amRegistrationResponse.getQueue();
@@ -437,6 +621,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
           if (uamResponse.getResponse() == null
               || !uamResponse.getResponse().getIsUnregistered()) {
             failedToUnRegister = true;
+          } else if (getNMStateStore() != null) {
+            getNMStateStore().removeAMRMProxyAppContextEntry(
+                getApplicationContext().getApplicationAttemptId(),
+                NMSS_SECONDARY_SC_PREFIX + uamResponse.getSubClusterId());
           }
         } catch (Throwable e) {
           failedToUnRegister = true;
@@ -496,6 +684,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     }
   }
 
+  @VisibleForTesting
+  protected FederationRegistryClient getRegistryClient() {
+    return this.registryClient;
+  }
+
   /**
    * Create the UAM pool manager for secondary sub-clsuters. For unit test to
    * override.
@@ -510,18 +703,20 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   }
 
   /**
-   * Returns instance of the ApplicationMasterProtocol proxy class that is used
-   * to connect to the Home resource manager.
+   * Create a proxy instance that is used to connect to the Home resource
+   * manager.
    *
    * @param appContext AMRMProxyApplicationContext
+   * @param protocol the protocol class for the proxy
+   * @param user the ugi for the proxy
+   * @param <T> the type of the proxy
    * @return the proxy created
    */
-  protected ApplicationMasterProtocol createHomeRMProxy(
-      AMRMProxyApplicationContext appContext) {
+  protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext,
+      Class<T> protocol, UserGroupInformation user) {
     try {
       return FederationProxyProviderUtil.createRMProxy(appContext.getConf(),
-          ApplicationMasterProtocol.class, this.homeSubClusterId, this.appOwner,
-          appContext.getAMRMToken());
+          protocol, this.homeSubClusterId, user, appContext.getAMRMToken());
     } catch (Exception ex) {
       throw new YarnRuntimeException(ex);
     }
@@ -810,17 +1005,31 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
                 responses.add(response);
               }
 
-              // Save the new AMRMToken for the UAM in registry if present
+              // Save the new AMRMToken for the UAM if present
               if (response.getAMRMToken() != null) {
                 Token<AMRMTokenIdentifier> newToken = ConverterUtils
                     .convertFromYarn(response.getAMRMToken(), (Text) null);
-                // Update the token in registry
+                // Update the token in registry or NMSS
                 if (registryClient != null) {
                   registryClient
                       .writeAMRMTokenForUAM(
                           getApplicationContext().getApplicationAttemptId()
                               .getApplicationId(),
                           subClusterId.getId(), newToken);
+                } else if (getNMStateStore() != null) {
+                  try {
+                    getNMStateStore().storeAMRMProxyAppContextEntry(
+                        getApplicationContext().getApplicationAttemptId(),
+                        NMSS_SECONDARY_SC_PREFIX + subClusterId.getId(),
+                        newToken.encodeToUrlString()
+                            .getBytes(STRING_TO_BYTE_FORMAT));
+                  } catch (IOException e) {
+                    LOG.error(
+                        "Error storing UAM token as AMRMProxy "
+                            + "context entry in NMSS for "
+                            + getApplicationContext().getApplicationAttemptId(),
+                        e);
+                  }
                 }
               }
 
@@ -925,12 +1134,20 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
             successfulRegistrations.put(uamResponse.getSubClusterId(),
                 uamResponse.getResponse());
 
+            // Save the UAM token in registry or NMSS
             if (registryClient != null) {
               registryClient.writeAMRMTokenForUAM(
                   getApplicationContext().getApplicationAttemptId()
                       .getApplicationId(),
                   uamResponse.getSubClusterId().getId(),
                   uamResponse.getUamToken());
+            } else if (getNMStateStore() != null) {
+              getNMStateStore().storeAMRMProxyAppContextEntry(
+                  getApplicationContext().getApplicationAttemptId(),
+                  NMSS_SECONDARY_SC_PREFIX
+                      + uamResponse.getSubClusterId().getId(),
+                  uamResponse.getUamToken().encodeToUrlString()
+                      .getBytes(STRING_TO_BYTE_FORMAT));
             }
           }
         } catch (Exception e) {
@@ -952,11 +1169,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   private AllocateResponse mergeAllocateResponses(
       AllocateResponse homeResponse) {
     // Timing issue, we need to remove the completed and then save the new ones.
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Remove containers: "
-          + homeResponse.getCompletedContainersStatuses());
-      LOG.debug("Adding containers: " + homeResponse.getAllocatedContainers());
-    }
     removeFinishedContainersFromCache(
         homeResponse.getCompletedContainersStatuses());
     cacheAllocatedContainers(homeResponse.getAllocatedContainers(),
@@ -989,6 +1201,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   private void removeFinishedContainersFromCache(
       List<ContainerStatus> finishedContainers) {
     for (ContainerStatus container : finishedContainers) {
+      LOG.debug("Completed container {}", container);
       if (containerIdToSubClusterIdMap
           .containsKey(container.getContainerId())) {
         containerIdToSubClusterIdMap.remove(container.getContainerId());
@@ -1146,12 +1359,22 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   private void cacheAllocatedContainers(List<Container> containers,
       SubClusterId subClusterId) {
     for (Container container : containers) {
+      LOG.debug("Adding container {}", container);
       if (containerIdToSubClusterIdMap.containsKey(container.getId())) {
         SubClusterId existingSubClusterId =
             containerIdToSubClusterIdMap.get(container.getId());
         if (existingSubClusterId.equals(subClusterId)) {
-          // When RM fails over, the new RM master might send out the same
-          // container allocation more than once. Just move on in this case.
+          /*
+           * When RM fails over, the new RM master might send out the same
+           * container allocation more than once.
+           *
+           * It is also possible because of a recent NM restart with NM recovery
+           * enabled. We recover running containers from RM. But RM might not
+           * notified AM of some of these containers yet. When RM dose notify,
+           * we will already have these containers in the map.
+           *
+           * Either case, just warn and move on.
+           */
           LOG.warn(
               "Duplicate containerID: {} found in the allocated containers"
                   + " from same sub-cluster: {}, so ignoring.",
@@ -1226,7 +1449,9 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    */
   private boolean warnIfNotExists(ContainerId containerId, String actionName) {
     if (!this.containerIdToSubClusterIdMap.containsKey(containerId)) {
-      LOG.error("AM is trying to {} a container {} that does not exist. ",
+      LOG.error(
+          "AM is trying to {} a container {} that does not exist. Might happen "
+              + "shortly after NM restart when NM recovery is enabled",
           actionName, containerId.toString());
       return false;
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index da1d047..0319dbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
 import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -180,6 +181,20 @@ public abstract class BaseAMRMProxyTest {
     return new NMContext(null, null, null, null, stateStore, false, this.conf);
   }
 
+  // A utility method for intercepter recover unit test
+  protected Map<String, byte[]> recoverDataMapForAppAttempt(
+      NMStateStoreService nmStateStore, ApplicationAttemptId attemptId)
+      throws IOException {
+    RecoveredAMRMProxyState state = nmStateStore.loadAMRMProxyState();
+    for (Map.Entry<ApplicationAttemptId, Map<String, byte[]>> entry : state
+        .getAppContexts().entrySet()) {
+      if (entry.getKey().equals(attemptId)) {
+        return entry.getValue();
+      }
+    }
+    return null;
+  }
+
   protected List<ContainerId> getCompletedContainerIds(
       List<ContainerStatus> containerStatus) {
     List<ContainerId> ret = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index aa7ed69..eefaba1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -450,6 +450,104 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
   }
 
   @Test
+  public void testRecoverWithAMRMProxyHA() throws Exception {
+    testRecover(registry);
+  }
+
+  @Test
+  public void testRecoverWithoutAMRMProxyHA() throws Exception {
+    testRecover(null);
+  }
+
+  public void testRecover(RegistryOperations registryObj) throws Exception {
+    ApplicationUserInfo userInfo = getApplicationUserInfo(testAppId);
+    userInfo.getUser().doAs(new PrivilegedExceptionAction<Object>() {
+      @Override
+      public Object run() throws Exception {
+        interceptor = new TestableFederationInterceptor();
+        interceptor.init(new AMRMProxyApplicationContextImpl(nmContext,
+            getConf(), attemptId, "test-user", null, null, null, registryObj));
+        interceptor.cleanupRegistry();
+
+        // Register the application
+        RegisterApplicationMasterRequest registerReq =
+            Records.newRecord(RegisterApplicationMasterRequest.class);
+        registerReq.setHost(Integer.toString(testAppId));
+        registerReq.setRpcPort(testAppId);
+        registerReq.setTrackingUrl("");
+
+        RegisterApplicationMasterResponse registerResponse =
+            interceptor.registerApplicationMaster(registerReq);
+        Assert.assertNotNull(registerResponse);
+
+        Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+        // Allocate one batch of containers
+        registerSubCluster(SubClusterId.newInstance("SC-1"));
+        registerSubCluster(SubClusterId.newInstance(HOME_SC_ID));
+
+        int numberOfContainers = 3;
+        List<Container> containers =
+            getContainersAndAssert(numberOfContainers, numberOfContainers * 2);
+        Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+
+        // Prepare for Federation Interceptor restart and recover
+        Map<String, byte[]> recoveredDataMap =
+            recoverDataMapForAppAttempt(nmStateStore, attemptId);
+        String scEntry =
+            FederationInterceptor.NMSS_SECONDARY_SC_PREFIX + "SC-1";
+        if (registryObj == null) {
+          Assert.assertTrue(recoveredDataMap.containsKey(scEntry));
+        } else {
+          // When AMRMPRoxy HA is enabled, NMSS should not have the UAM token,
+          // it should be in Registry
+          Assert.assertFalse(recoveredDataMap.containsKey(scEntry));
+        }
+
+        // Preserve the mock RM instances
+        MockResourceManagerFacade homeRM = interceptor.getHomeRM();
+        ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
+            interceptor.getSecondaryRMs();
+
+        // Create a new intercepter instance and recover
+        interceptor = new TestableFederationInterceptor(homeRM, secondaries);
+        interceptor.init(new AMRMProxyApplicationContextImpl(nmContext,
+            getConf(), attemptId, "test-user", null, null, null, registryObj));
+        interceptor.recover(recoveredDataMap);
+
+        Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+
+        // Release all containers
+        releaseContainersAndAssert(containers);
+
+        // Finish the application
+        FinishApplicationMasterRequest finishReq =
+            Records.newRecord(FinishApplicationMasterRequest.class);
+        finishReq.setDiagnostics("");
+        finishReq.setTrackingUrl("");
+        finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+        FinishApplicationMasterResponse finshResponse =
+            interceptor.finishApplicationMaster(finishReq);
+        Assert.assertNotNull(finshResponse);
+        Assert.assertEquals(true, finshResponse.getIsUnregistered());
+
+        // After the application succeeds, the registry/NMSS entry should be
+        // cleaned up
+        if (registryObj != null) {
+          Assert.assertEquals(0,
+              interceptor.getRegistryClient().getAllApplications().size());
+        } else {
+          recoveredDataMap =
+              recoverDataMapForAppAttempt(nmStateStore, attemptId);
+          Assert.assertFalse(recoveredDataMap.containsKey(scEntry));
+        }
+        return null;
+      }
+    });
+  }
+
+  @Test
   public void testRequestInterceptorChainCreation() throws Exception {
     RequestInterceptor root =
         super.getAMRMProxyService().createRequestInterceptorChain();
@@ -636,6 +734,12 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
             interceptor.finishApplicationMaster(finishReq);
         Assert.assertNotNull(finshResponse);
         Assert.assertEquals(true, finshResponse.getIsUnregistered());
+
+        // After the application succeeds, the registry entry should be deleted
+        if (interceptor.getRegistryClient() != null) {
+          Assert.assertEquals(0,
+              interceptor.getRegistryClient().getAllApplications().size());
+        }
         return null;
       }
     });

http://git-wip-us.apache.org/repos/asf/hadoop/blob/670e8d4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
index 23c80ae..1088c69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
@@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -59,16 +58,17 @@ public class TestableFederationInterceptor extends FederationInterceptor {
     return new TestableUnmanagedAMPoolManager(threadPool);
   }
 
+  @SuppressWarnings("unchecked")
   @Override
-  protected ApplicationMasterProtocol createHomeRMProxy(
-      AMRMProxyApplicationContext appContext) {
+  protected <T> T createHomeRMProxy(AMRMProxyApplicationContext appContext,
+      Class<T> protocol, UserGroupInformation user) {
     synchronized (this) {
       if (mockRm == null) {
         mockRm = new MockResourceManagerFacade(
             new YarnConfiguration(super.getConf()), 0);
       }
     }
-    return mockRm;
+    return (T) mockRm;
   }
 
   @SuppressWarnings("unchecked")


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[29/50] hadoop git commit: YARN-7473. Implement Framework and policy for capacity management of auto created queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
new file mode 100644
index 0000000..f6b4f2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for dynamic auto created leaf queues.
+ * @see ReservationQueue
+ */
+public class TestReservationQueue {
+
+  private CapacitySchedulerConfiguration csConf;
+  private CapacitySchedulerContext csContext;
+  final static int DEF_MAX_APPS = 10000;
+  final static int GB = 1024;
+  private final ResourceCalculator resourceCalculator =
+      new DefaultResourceCalculator();
+  private ReservationQueue autoCreatedLeafQueue;
+
+  @Before
+  public void setup() throws IOException {
+    // setup a context / conf
+    csConf = new CapacitySchedulerConfiguration();
+
+    YarnConfiguration conf = new YarnConfiguration();
+    csContext = mock(CapacitySchedulerContext.class);
+    when(csContext.getConfiguration()).thenReturn(csConf);
+    when(csContext.getConf()).thenReturn(conf);
+    when(csContext.getMinimumResourceCapability()).thenReturn(
+        Resources.createResource(GB, 1));
+    when(csContext.getMaximumResourceCapability()).thenReturn(
+        Resources.createResource(16 * GB, 32));
+    when(csContext.getClusterResource()).thenReturn(
+        Resources.createResource(100 * 16 * GB, 100 * 32));
+    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
+    RMContext mockRMContext = TestUtils.getMockRMContext();
+    when(csContext.getRMContext()).thenReturn(mockRMContext);
+
+    // create a queue
+    PlanQueue pq = new PlanQueue(csContext, "root", null, null);
+    autoCreatedLeafQueue = new ReservationQueue(csContext, "a", pq);
+  }
+
+  private void validateAutoCreatedLeafQueue(double capacity) {
+    assertTrue(" actual capacity: " + autoCreatedLeafQueue.getCapacity(),
+        autoCreatedLeafQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
+    assertEquals(autoCreatedLeafQueue.maxApplications, DEF_MAX_APPS);
+    assertEquals(autoCreatedLeafQueue.maxApplicationsPerUser, DEF_MAX_APPS);
+  }
+
+  @Test
+  public void testAddSubtractCapacity() throws Exception {
+
+    // verify that setting, adding, subtracting capacity works
+    autoCreatedLeafQueue.setCapacity(1.0F);
+    validateAutoCreatedLeafQueue(1);
+    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
+    validateAutoCreatedLeafQueue(0.9);
+    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1f, 1f));
+    validateAutoCreatedLeafQueue(1);
+    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0f, 1f));
+    validateAutoCreatedLeafQueue(0);
+
+    try {
+      autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
+      fail();
+    } catch (SchedulerDynamicEditException iae) {
+      // expected
+      validateAutoCreatedLeafQueue(1);
+    }
+
+    try {
+      autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(-0.1f, 1f));
+      fail();
+    } catch (SchedulerDynamicEditException iae) {
+      // expected
+      validateAutoCreatedLeafQueue(1);
+    }
+
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[30/50] hadoop git commit: YARN-7473. Implement Framework and policy for capacity management of auto created queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
new file mode 100644
index 0000000..4e77339
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -0,0 +1,579 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.placement
+    .ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement
+    .UserGroupMappingPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .ResourceScheduler;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
+    .QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .SchedulerEvent;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
+    .NO_LABEL;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CSQueueUtils.EPSILON;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestCapacitySchedulerAutoCreatedQueueBase {
+
+  private static final Log LOG = LogFactory.getLog(
+      TestCapacitySchedulerAutoCreatedQueueBase.class);
+  public final int GB = 1024;
+  public final static ContainerUpdates NULL_UPDATE_REQUESTS =
+      new ContainerUpdates();
+
+  public static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+  public static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+  public static final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+  public static final String D = CapacitySchedulerConfiguration.ROOT + ".d";
+  public static final String A1 = A + ".a1";
+  public static final String A2 = A + ".a2";
+  public static final String B1 = B + ".b1";
+  public static final String B2 = B + ".b2";
+  public static final String B3 = B + ".b3";
+  public static final String C1 = C + ".c1";
+  public static final String C2 = C + ".c2";
+  public static final String C3 = C + ".c3";
+  public static final float A_CAPACITY = 20f;
+  public static final float B_CAPACITY = 40f;
+  public static final float C_CAPACITY = 20f;
+  public static final float D_CAPACITY = 20f;
+  public static final float A1_CAPACITY = 30;
+  public static final float A2_CAPACITY = 70;
+  public static final float B1_CAPACITY = 60f;
+  public static final float B2_CAPACITY = 20f;
+  public static final float B3_CAPACITY = 20f;
+  public static final float C1_CAPACITY = 20f;
+  public static final float C2_CAPACITY = 20f;
+
+  public static final String USER = "user_";
+  public static final String USER0 = USER + 0;
+  public static final String USER1 = USER + 1;
+  public static final String USER3 = USER + 3;
+  public static final String USER2 = USER + 2;
+  public static final String PARENT_QUEUE = "c";
+
+  public static final Set<String> accessibleNodeLabelsOnC = new HashSet<>();
+
+  public static final String NODEL_LABEL_GPU = "GPU";
+  public static final String NODEL_LABEL_SSD = "SSD";
+
+  protected MockRM mockRM = null;
+  protected CapacityScheduler cs;
+  private final TestCapacityScheduler tcs = new TestCapacityScheduler();
+  protected SpyDispatcher dispatcher;
+  private static EventHandler<Event> rmAppEventEventHandler;
+
+  public static class SpyDispatcher extends AsyncDispatcher {
+
+    public static BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<>();
+
+    public static class SpyRMAppEventHandler implements EventHandler<Event> {
+      public void handle(Event event) {
+        eventQueue.add(event);
+      }
+    }
+
+    @Override
+    protected void dispatch(Event event) {
+      eventQueue.add(event);
+    }
+
+    @Override
+    public EventHandler<Event> getEventHandler() {
+      return rmAppEventEventHandler;
+    }
+
+    void spyOnNextEvent(Event expectedEvent, long timeout)
+        throws InterruptedException {
+
+      Event event = eventQueue.poll(timeout, TimeUnit.MILLISECONDS);
+      assertEquals(expectedEvent.getType(), event.getType());
+      assertEquals(expectedEvent.getClass(), event.getClass());
+    }
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    setupQueueMappings(conf);
+
+    mockRM = new MockRM(conf);
+    cs = (CapacityScheduler) mockRM.getResourceScheduler();
+
+    dispatcher = new SpyDispatcher();
+    rmAppEventEventHandler = new SpyDispatcher.SpyRMAppEventHandler();
+    dispatcher.register(RMAppEventType.class, rmAppEventEventHandler);
+    cs.updatePlacementRules();
+    mockRM.start();
+
+    cs.start();
+  }
+
+  public static CapacitySchedulerConfiguration setupQueueMappings(
+      CapacitySchedulerConfiguration conf) {
+
+    List<String> queuePlacementRules = new ArrayList<>();
+    queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
+    conf.setQueuePlacementRules(queuePlacementRules);
+
+    //set queue mapping
+    List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
+        new ArrayList<>();
+    for (int i = 0; i <= 3; i++) {
+      //Set C as parent queue name for auto queue creation
+      UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
+          new UserGroupMappingPlacementRule.QueueMapping(
+              UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
+              USER + i, getQueueMapping(PARENT_QUEUE, USER + i));
+      queueMappings.add(userQueueMapping);
+    }
+
+    conf.setQueueMappings(queueMappings);
+    //override with queue mappings
+    conf.setOverrideWithQueueMappings(true);
+    return conf;
+  }
+
+  /**
+   * @param conf, to be modified
+   * @return, CS configuration which has C as an auto creation enabled parent
+   * queue
+   * <p>
+   * root /     \      \       \ a        b      c    d / \    /  |  \ a1  a2 b1
+   * b2  b3
+   */
+  public static CapacitySchedulerConfiguration setupQueueConfiguration(
+      CapacitySchedulerConfiguration conf) {
+
+    //setup new queues with one of them auto enabled
+    // Define top-level queues
+    // Set childQueue for root
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] { "a", "b", "c", "d" });
+
+    conf.setCapacity(A, A_CAPACITY);
+    conf.setCapacity(B, B_CAPACITY);
+    conf.setCapacity(C, C_CAPACITY);
+    conf.setCapacity(D, D_CAPACITY);
+
+    // Define 2nd-level queues
+    conf.setQueues(A, new String[] { "a1", "a2" });
+    conf.setCapacity(A1, A1_CAPACITY);
+    conf.setUserLimitFactor(A1, 100.0f);
+    conf.setCapacity(A2, A2_CAPACITY);
+    conf.setUserLimitFactor(A2, 100.0f);
+
+    conf.setQueues(B, new String[] { "b1", "b2", "b3" });
+    conf.setCapacity(B1, B1_CAPACITY);
+    conf.setUserLimitFactor(B1, 100.0f);
+    conf.setCapacity(B2, B2_CAPACITY);
+    conf.setUserLimitFactor(B2, 100.0f);
+    conf.setCapacity(B3, B3_CAPACITY);
+    conf.setUserLimitFactor(B3, 100.0f);
+
+    conf.setUserLimitFactor(C, 1.0f);
+    conf.setAutoCreateChildQueueEnabled(C, true);
+
+    //Setup leaf queue template configs
+    conf.setAutoCreatedLeafQueueConfigCapacity(C, 50.0f);
+    conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f);
+    conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100);
+    conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f);
+
+    LOG.info("Setup " + C + " as an auto leaf creation enabled parent queue");
+
+    conf.setUserLimitFactor(D, 1.0f);
+    conf.setAutoCreateChildQueueEnabled(D, true);
+    conf.setUserLimit(D, 100);
+    conf.setUserLimitFactor(D, 3.0f);
+
+    //Setup leaf queue template configs
+    conf.setAutoCreatedLeafQueueConfigCapacity(D, 10.0f);
+    conf.setAutoCreatedLeafQueueConfigMaxCapacity(D, 100.0f);
+    conf.setAutoCreatedLeafQueueConfigUserLimit(D, 3);
+    conf.setAutoCreatedLeafQueueConfigUserLimitFactor(D, 100);
+
+    conf.set(CapacitySchedulerConfiguration.PREFIX + C + DOT
+            + CapacitySchedulerConfiguration
+            .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+            + DOT + CapacitySchedulerConfiguration.ORDERING_POLICY,
+        FAIR_APP_ORDERING_POLICY);
+
+    accessibleNodeLabelsOnC.add(NODEL_LABEL_GPU);
+    accessibleNodeLabelsOnC.add(NODEL_LABEL_SSD);
+    accessibleNodeLabelsOnC.add(NO_LABEL);
+
+    conf.setAccessibleNodeLabels(C, accessibleNodeLabelsOnC);
+    conf.setCapacityByLabel(C, NODEL_LABEL_GPU, 50);
+    conf.setCapacityByLabel(C, NODEL_LABEL_SSD, 50);
+
+    LOG.info("Setup " + D + " as an auto leaf creation enabled parent queue");
+
+    return conf;
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (mockRM != null) {
+      mockRM.stop();
+    }
+  }
+
+  protected void validateCapacities(AutoCreatedLeafQueue autoCreatedLeafQueue,
+      float capacity, float absCapacity, float maxCapacity,
+      float absMaxCapacity) {
+    assertEquals(capacity, autoCreatedLeafQueue.getCapacity(), EPSILON);
+    assertEquals(absCapacity, autoCreatedLeafQueue.getAbsoluteCapacity(),
+        EPSILON);
+    assertEquals(maxCapacity, autoCreatedLeafQueue.getMaximumCapacity(),
+        EPSILON);
+    assertEquals(absMaxCapacity,
+        autoCreatedLeafQueue.getAbsoluteMaximumCapacity(), EPSILON);
+  }
+
+  protected void cleanupQueue(String queueName) throws YarnException {
+    AutoCreatedLeafQueue queue = (AutoCreatedLeafQueue) cs.getQueue(queueName);
+    if (queue != null) {
+      setEntitlement(queue, new QueueEntitlement(0.0f, 0.0f));
+      ((ManagedParentQueue) queue.getParent()).removeChildQueue(
+          queue.getQueueName());
+      cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueueName());
+    }
+  }
+
+  protected ApplicationId submitApp(MockRM rm, CSQueue parentQueue,
+      String leafQueueName, String user, int expectedNumAppsInParentQueue,
+      int expectedNumAppsInLeafQueue) throws Exception {
+
+    CapacityScheduler capacityScheduler =
+        (CapacityScheduler) rm.getResourceScheduler();
+    // submit an app
+    RMApp rmApp = rm.submitApp(GB, "test-auto-queue-activation", user, null,
+        leafQueueName);
+
+    // check preconditions
+    List<ApplicationAttemptId> appsInParentQueue =
+        capacityScheduler.getAppsInQueue(parentQueue.getQueueName());
+    assertEquals(expectedNumAppsInParentQueue, appsInParentQueue.size());
+
+    List<ApplicationAttemptId> appsInLeafQueue =
+        capacityScheduler.getAppsInQueue(leafQueueName);
+    assertEquals(expectedNumAppsInLeafQueue, appsInLeafQueue.size());
+
+    return rmApp.getApplicationId();
+  }
+
+  protected List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping(
+      CapacityScheduler newCS, String user, String parentQueue, String queue) {
+    List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
+        new ArrayList<>();
+    queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping(
+        UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user,
+        getQueueMapping(parentQueue, queue)));
+    newCS.getConfiguration().setQueueMappings(queueMappings);
+    return queueMappings;
+  }
+
+  protected MockRM setupSchedulerInstance() {
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    List<String> queuePlacementRules = new ArrayList<String>();
+    queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
+    conf.setQueuePlacementRules(queuePlacementRules);
+
+    setupQueueMappings(conf);
+
+    MockRM newMockRM = new MockRM(conf);
+    newMockRM.start();
+    ((CapacityScheduler) newMockRM.getResourceScheduler()).start();
+    return newMockRM;
+  }
+
+  protected void checkQueueCapacities(CapacityScheduler newCS, float capacityC,
+      float capacityD) {
+    CSQueue rootQueue = newCS.getRootQueue();
+    CSQueue queueC = tcs.findQueue(rootQueue, C);
+    CSQueue queueD = tcs.findQueue(rootQueue, D);
+    CSQueue queueC1 = tcs.findQueue(queueC, C1);
+    CSQueue queueC2 = tcs.findQueue(queueC, C2);
+    CSQueue queueC3 = tcs.findQueue(queueC, C3);
+
+    float capC = capacityC / 100.0f;
+    float capD = capacityD / 100.0f;
+
+    tcs.checkQueueCapacity(queueC, capC, capC, 1.0f, 1.0f);
+    tcs.checkQueueCapacity(queueD, capD, capD, 1.0f, 1.0f);
+    tcs.checkQueueCapacity(queueC1, C1_CAPACITY / 100.0f,
+        (C1_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
+    tcs.checkQueueCapacity(queueC2, C2_CAPACITY / 100.0f,
+        (C2_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
+
+    if (queueC3 != null) {
+      ManagedParentQueue parentQueue = (ManagedParentQueue) queueC;
+      QueueCapacities cap =
+          parentQueue.getLeafQueueTemplate().getQueueCapacities();
+      tcs.checkQueueCapacity(queueC3, cap.getCapacity(),
+          (cap.getCapacity()) * capC, 1.0f, 1.0f);
+    }
+  }
+
+  static String getQueueMapping(String parentQueue, String leafQueue) {
+    return parentQueue + DOT + leafQueue;
+  }
+
+  protected ApplicationAttemptId submitApp(CapacityScheduler newCS, String user,
+      String queue, String parentQueue) {
+    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, queue, user,
+        new ApplicationPlacementContext(queue, parentQueue));
+    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+        appId, 1);
+    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(
+        appAttemptId, false);
+    newCS.handle(addAppEvent);
+    newCS.handle(addAttemptEvent);
+    return appAttemptId;
+  }
+
+  protected RMApp submitApp(String user, String queue, String nodeLabel)
+      throws Exception {
+    RMApp app = mockRM.submitApp(GB,
+        "test-auto-queue-creation" + RandomUtils.nextInt(100), user, null,
+        queue, nodeLabel);
+    Assert.assertEquals(app.getAmNodeLabelExpression(), nodeLabel);
+    // check preconditions
+    List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
+    assertEquals(1, appsInC.size());
+    assertNotNull(cs.getQueue(queue));
+
+    return app;
+  }
+
+  void setEntitlement(AutoCreatedLeafQueue queue,
+      QueueEntitlement entitlement) {
+    queue.setCapacity(entitlement.getCapacity());
+    queue.setAbsoluteCapacity(
+        queue.getParent().getAbsoluteCapacity() * entitlement.getCapacity());
+    // note: we currently set maxCapacity to capacity
+    // this might be revised later
+    queue.setMaxCapacity(entitlement.getMaxCapacity());
+  }
+
+  protected void validateUserAndAppLimits(
+      AutoCreatedLeafQueue autoCreatedLeafQueue, int maxApps,
+      int maxAppsPerUser) {
+    assertEquals(maxApps, autoCreatedLeafQueue.getMaxApplications());
+    assertEquals(maxAppsPerUser,
+        autoCreatedLeafQueue.getMaxApplicationsPerUser());
+  }
+
+  protected void validateInitialQueueEntitlement(CSQueue parentQueue,
+      String leafQueueName, float expectedTotalChildQueueAbsCapacity)
+      throws SchedulerDynamicEditException {
+    validateInitialQueueEntitlement(cs, parentQueue, leafQueueName,
+        expectedTotalChildQueueAbsCapacity);
+  }
+
+  protected void validateInitialQueueEntitlement(
+      CapacityScheduler capacityScheduler, CSQueue parentQueue,
+      String leafQueueName, float expectedTotalChildQueueAbsCapacity)
+      throws SchedulerDynamicEditException {
+    ManagedParentQueue autoCreateEnabledParentQueue =
+        (ManagedParentQueue) parentQueue;
+
+    GuaranteedOrZeroCapacityOverTimePolicy policy =
+        (GuaranteedOrZeroCapacityOverTimePolicy) autoCreateEnabledParentQueue
+            .getAutoCreatedQueueManagementPolicy();
+
+    assertEquals(expectedTotalChildQueueAbsCapacity,
+        policy.getAbsoluteActivatedChildQueueCapacity(), EPSILON);
+
+    AutoCreatedLeafQueue leafQueue =
+        (AutoCreatedLeafQueue) capacityScheduler.getQueue(leafQueueName);
+
+    for (String label : accessibleNodeLabelsOnC) {
+      validateCapacitiesByLabel(autoCreateEnabledParentQueue, leafQueue, label);
+    }
+
+    assertEquals(true, policy.isActive(leafQueue));
+  }
+
+  protected void validateCapacitiesByLabel(
+      ManagedParentQueue autoCreateEnabledParentQueue,
+      AutoCreatedLeafQueue leafQueue, String label) {
+    assertEquals(
+        autoCreateEnabledParentQueue.getLeafQueueTemplate().getQueueCapacities()
+            .getCapacity(), leafQueue.getQueueCapacities().getCapacity(label),
+        EPSILON);
+    assertEquals(
+        autoCreateEnabledParentQueue.getLeafQueueTemplate().getQueueCapacities()
+            .getMaximumCapacity(),
+        leafQueue.getQueueCapacities().getMaximumCapacity(label), EPSILON);
+  }
+
+  protected void validateActivatedQueueEntitlement(CSQueue parentQueue,
+      String leafQueueName, float expectedTotalChildQueueAbsCapacity,
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException {
+    ManagedParentQueue autoCreateEnabledParentQueue =
+        (ManagedParentQueue) parentQueue;
+
+    GuaranteedOrZeroCapacityOverTimePolicy policy =
+        (GuaranteedOrZeroCapacityOverTimePolicy) autoCreateEnabledParentQueue
+            .getAutoCreatedQueueManagementPolicy();
+
+    QueueCapacities cap = autoCreateEnabledParentQueue.getLeafQueueTemplate()
+        .getQueueCapacities();
+    QueueEntitlement expectedEntitlement = new QueueEntitlement(
+        cap.getCapacity(), cap.getMaximumCapacity());
+
+    //validate capacity
+    validateQueueEntitlements(leafQueueName, expectedEntitlement,
+        queueManagementChanges);
+
+    //validate parent queue state
+    assertEquals(expectedTotalChildQueueAbsCapacity,
+        policy.getAbsoluteActivatedChildQueueCapacity(), EPSILON);
+
+    AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
+        leafQueueName);
+
+    //validate leaf queue state
+    assertEquals(true, policy.isActive(leafQueue));
+  }
+
+  protected void validateDeactivatedQueueEntitlement(CSQueue parentQueue,
+      String leafQueueName, float expectedTotalChildQueueAbsCapacity,
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException {
+    QueueEntitlement expectedEntitlement = new QueueEntitlement(0.0f, 1.0f);
+
+    ManagedParentQueue autoCreateEnabledParentQueue =
+        (ManagedParentQueue) parentQueue;
+
+    AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
+        leafQueueName);
+
+    GuaranteedOrZeroCapacityOverTimePolicy policy =
+        (GuaranteedOrZeroCapacityOverTimePolicy) autoCreateEnabledParentQueue
+            .getAutoCreatedQueueManagementPolicy();
+
+    //validate parent queue state
+    assertEquals(expectedTotalChildQueueAbsCapacity,
+        policy.getAbsoluteActivatedChildQueueCapacity(), EPSILON);
+
+    //validate leaf queue state
+    assertEquals(false, policy.isActive(leafQueue));
+
+    //validate capacity
+    validateQueueEntitlements(leafQueueName, expectedEntitlement,
+        queueManagementChanges);
+  }
+
+  private void validateQueueEntitlements(String leafQueueName,
+      QueueEntitlement expectedEntitlement,
+      List<QueueManagementChange> queueEntitlementChanges) {
+    AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
+        leafQueueName);
+    validateQueueEntitlementChangesForLeafQueue(leafQueue, expectedEntitlement,
+        queueEntitlementChanges);
+  }
+
+  private void validateQueueEntitlementChangesForLeafQueue(CSQueue leafQueue,
+      QueueEntitlement expectedQueueEntitlement,
+      final List<QueueManagementChange> queueEntitlementChanges) {
+    boolean found = false;
+    for (QueueManagementChange entitlementChange : queueEntitlementChanges) {
+      if (leafQueue.getQueueName().equals(
+          entitlementChange.getQueue().getQueueName())) {
+
+        AutoCreatedLeafQueueConfig updatedQueueTemplate =
+            entitlementChange.getUpdatedQueueTemplate();
+
+        for (String label : accessibleNodeLabelsOnC) {
+          QueueEntitlement newEntitlement = new QueueEntitlement(
+              updatedQueueTemplate.getQueueCapacities().getCapacity(label),
+              updatedQueueTemplate.getQueueCapacities()
+                  .getMaximumCapacity(label));
+          assertEquals(expectedQueueEntitlement, newEntitlement);
+        }
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      fail("Could not find the specified leaf queue in entitlement changes : "
+          + leafQueue.getQueueName());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 7090bc9..049a932 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -26,51 +26,54 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Event;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.placement
+    .ApplicationPlacementContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
+    .RMAppAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .SchedulerDynamicEditException;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
+    .QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .SchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy
+    .FairOrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.security
+    .ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security
+    .NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security
+    .RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.After;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
 
-import static org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.CURRENT_USER_MAPPING;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.placement
+    .UserGroupMappingPlacementRule.CURRENT_USER_MAPPING;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CSQueueUtils.EPSILON;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -79,198 +82,14 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 /**
- * Tests for creation and reinitilization of auto created leaf queues
+ * Tests for creation and reinitialization of auto created leaf queues
  * under a ManagedParentQueue.
  */
-public class TestCapacitySchedulerAutoQueueCreation {
-
-  private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
-  private final int GB = 1024;
-  private final static ContainerUpdates NULL_UPDATE_REQUESTS =
-      new ContainerUpdates();
-
-  private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
-  private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
-  private static final String C = CapacitySchedulerConfiguration.ROOT + ".c";
-  private static final String D = CapacitySchedulerConfiguration.ROOT + ".d";
-  private static final String A1 = A + ".a1";
-  private static final String A2 = A + ".a2";
-  private static final String B1 = B + ".b1";
-  private static final String B2 = B + ".b2";
-  private static final String B3 = B + ".b3";
-  private static final String C1 = C + ".c1";
-  private static final String C2 = C + ".c2";
-  private static final String C3 = C + ".c3";
-  private static float A_CAPACITY = 20f;
-  private static float B_CAPACITY = 40f;
-  private static float C_CAPACITY = 20f;
-  private static float D_CAPACITY = 20f;
-  private static float A1_CAPACITY = 30;
-  private static float A2_CAPACITY = 70;
-  private static float B1_CAPACITY = 60f;
-  private static float B2_CAPACITY = 20f;
-  private static float B3_CAPACITY = 20f;
-  private static float C1_CAPACITY = 20f;
-  private static float C2_CAPACITY = 20f;
-
-  private static String USER = "user_";
-  private static String USER0 = USER + 0;
-  private static String USER2 = USER + 2;
-  private static String PARENT_QUEUE = "c";
-
-  private MockRM mockRM = null;
-
-  private CapacityScheduler cs;
-
-  private final TestCapacityScheduler tcs = new TestCapacityScheduler();
-
-  private static SpyDispatcher dispatcher;
-
-  private static EventHandler<Event> rmAppEventEventHandler;
-
-  private static class SpyDispatcher extends AsyncDispatcher {
-
-    private static BlockingQueue<Event> eventQueue =
-        new LinkedBlockingQueue<>();
-
-    private static class SpyRMAppEventHandler implements EventHandler<Event> {
-      public void handle(Event event) {
-        eventQueue.add(event);
-      }
-    }
-
-    @Override
-    protected void dispatch(Event event) {
-      eventQueue.add(event);
-    }
-
-    @Override
-    public EventHandler<Event> getEventHandler() {
-      return rmAppEventEventHandler;
-    }
-
-    void spyOnNextEvent(Event expectedEvent, long timeout)
-        throws InterruptedException {
-
-      Event event = eventQueue.poll(timeout, TimeUnit.MILLISECONDS);
-      assertEquals(expectedEvent.getType(), event.getType());
-      assertEquals(expectedEvent.getClass(), event.getClass());
-    }
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
-
-    List<String> queuePlacementRules = new ArrayList<>();
-    queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
-    conf.setQueuePlacementRules(queuePlacementRules);
-
-    setupQueueMappings(conf);
-
-    mockRM = new MockRM(conf);
-    cs = (CapacityScheduler) mockRM.getResourceScheduler();
-
-    dispatcher = new SpyDispatcher();
-    rmAppEventEventHandler = new SpyDispatcher.SpyRMAppEventHandler();
-    dispatcher.register(RMAppEventType.class, rmAppEventEventHandler);
-    cs.updatePlacementRules();
-    mockRM.start();
-
-    cs.start();
-  }
-
-  private CapacitySchedulerConfiguration setupQueueMappings(
-      CapacitySchedulerConfiguration conf) {
-
-    //set queue mapping
-    List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
-        new ArrayList<>();
-    for (int i = 0; i <= 3; i++) {
-      //Set C as parent queue name for auto queue creation
-      UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
-          new UserGroupMappingPlacementRule.QueueMapping(
-              UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
-              USER + i, getQueueMapping(PARENT_QUEUE, USER + i));
-      queueMappings.add(userQueueMapping);
-    }
+public class TestCapacitySchedulerAutoQueueCreation
+    extends TestCapacitySchedulerAutoCreatedQueueBase {
 
-    conf.setQueueMappings(queueMappings);
-    //override with queue mappings
-    conf.setOverrideWithQueueMappings(true);
-    return conf;
-  }
-
-  /**
-   * @param conf, to be modified
-   * @return, CS configuration which has C
-   * as an auto creation enabled parent queue
-   * <p>
-   * root
-   * /     \      \       \
-   * a        b      c    d
-   * / \    /  |  \
-   * a1  a2 b1  b2  b3
-   */
-  private CapacitySchedulerConfiguration setupQueueConfiguration(
-      CapacitySchedulerConfiguration conf) {
-
-    //setup new queues with one of them auto enabled
-    // Define top-level queues
-    // Set childQueue for root
-    conf.setQueues(CapacitySchedulerConfiguration.ROOT,
-        new String[] { "a", "b", "c", "d" });
-
-    conf.setCapacity(A, A_CAPACITY);
-    conf.setCapacity(B, B_CAPACITY);
-    conf.setCapacity(C, C_CAPACITY);
-    conf.setCapacity(D, D_CAPACITY);
-
-    // Define 2nd-level queues
-    conf.setQueues(A, new String[] { "a1", "a2" });
-    conf.setCapacity(A1, A1_CAPACITY);
-    conf.setUserLimitFactor(A1, 100.0f);
-    conf.setCapacity(A2, A2_CAPACITY);
-    conf.setUserLimitFactor(A2, 100.0f);
-
-    conf.setQueues(B, new String[] { "b1", "b2", "b3" });
-    conf.setCapacity(B1, B1_CAPACITY);
-    conf.setUserLimitFactor(B1, 100.0f);
-    conf.setCapacity(B2, B2_CAPACITY);
-    conf.setUserLimitFactor(B2, 100.0f);
-    conf.setCapacity(B3, B3_CAPACITY);
-    conf.setUserLimitFactor(B3, 100.0f);
-
-    conf.setUserLimitFactor(C, 1.0f);
-    conf.setAutoCreateChildQueueEnabled(C, true);
-
-    //Setup leaf queue template configs
-    conf.setAutoCreatedLeafQueueTemplateCapacity(C, 50.0f);
-    conf.setAutoCreatedLeafQueueTemplateMaxCapacity(C, 100.0f);
-
-    LOG.info("Setup " + C + " as an auto leaf creation enabled parent queue");
-
-    conf.setUserLimitFactor(D, 1.0f);
-    conf.setAutoCreateChildQueueEnabled(D, true);
-
-    //Setup leaf queue template configs
-    conf.setAutoCreatedLeafQueueTemplateCapacity(D, 10.0f);
-    conf.setAutoCreatedLeafQueueTemplateMaxCapacity(D, 100.0f);
-
-    LOG.info("Setup " + D + " as an auto leaf creation enabled parent queue");
-
-    return conf;
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    if (mockRM != null) {
-      mockRM.stop();
-    }
-  }
+  private static final Log LOG = LogFactory.getLog(
+      TestCapacitySchedulerAutoQueueCreation.class);
 
   @Test(timeout = 10000)
   public void testAutoCreateLeafQueueCreation() throws Exception {
@@ -289,7 +108,11 @@ public class TestCapacitySchedulerAutoQueueCreation {
       ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(
           PARENT_QUEUE);
       assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
-      validateCapacities(autoCreatedLeafQueue);
+      validateInitialQueueEntitlement(parentQueue, USER0, 0.1f);
+      validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000);
+
+      assertTrue(autoCreatedLeafQueue
+          .getOrderingPolicy() instanceof FairOrderingPolicy);
     } finally {
       cleanupQueue(USER0);
     }
@@ -297,7 +120,6 @@ public class TestCapacitySchedulerAutoQueueCreation {
 
   @Test
   public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception {
-
     try {
       String host = "127.0.0.1";
       RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
@@ -306,20 +128,28 @@ public class TestCapacitySchedulerAutoQueueCreation {
 
       // submit an app
 
-      RMApp app = mockRM.submitApp(GB, "test-auto-queue-creation-1", USER0,
+      RMApp app1 = mockRM.submitApp(GB, "test-auto-queue-creation-1", USER0,
           null, USER0);
+
+      RMApp app2 = mockRM.submitApp(GB, "test-auto-queue-creation-2", USER1,
+          null, USER1);
       // check preconditions
       List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
-      assertEquals(1, appsInC.size());
+      assertEquals(2, appsInC.size());
 
       assertNotNull(cs.getQueue(USER0));
+      assertNotNull(cs.getQueue(USER1));
 
-      AutoCreatedLeafQueue autoCreatedLeafQueue =
-          (AutoCreatedLeafQueue) cs.getQueue(USER0);
+      AutoCreatedLeafQueue user0Queue = (AutoCreatedLeafQueue) cs.getQueue(
+          USER0);
+      AutoCreatedLeafQueue user1Queue = (AutoCreatedLeafQueue) cs.getQueue(
+          USER0);
       ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(
           PARENT_QUEUE);
-      assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
-      validateCapacities(autoCreatedLeafQueue);
+      assertEquals(parentQueue, user0Queue.getParent());
+      assertEquals(parentQueue, user1Queue.getParent());
+      validateInitialQueueEntitlement(parentQueue, USER0, 0.2f);
+      validateInitialQueueEntitlement(parentQueue, USER1, 0.2f);
 
       ApplicationAttemptId appAttemptId = appsInC.get(0);
 
@@ -337,7 +167,7 @@ public class TestCapacitySchedulerAutoQueueCreation {
       CapacityScheduler.schedule(cs);
 
       //change state to draining
-      autoCreatedLeafQueue.stopQueue();
+      user0Queue.stopQueue();
 
       cs.killAllAppsInQueue(USER0);
 
@@ -346,80 +176,24 @@ public class TestCapacitySchedulerAutoQueueCreation {
       mockRM.waitForState(appAttemptId.getApplicationId(), RMAppState.KILLED);
 
       //change state to stopped
-      autoCreatedLeafQueue.stopQueue();
+      user0Queue.stopQueue();
       assertEquals(QueueState.STOPPED,
-          autoCreatedLeafQueue.getQueueInfo().getQueueState());
+          user0Queue.getQueueInfo().getQueueState());
 
       cs.reinitialize(cs.getConf(), mockRM.getRMContext());
 
-      AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
-          USER0);
-      validateCapacities(leafQueue);
-
-    } finally {
-      cleanupQueue(USER0);
-    }
-  }
-
-  @Test
-  public void testRefreshQueuesWithAutoCreatedLeafQueues() throws Exception {
-
-    MockRM newMockRM = setupSchedulerInstance();
-    try {
-      CapacityScheduler newCS =
-          (CapacityScheduler) newMockRM.getResourceScheduler();
-      CapacitySchedulerConfiguration conf = newCS.getConfiguration();
-
-      // Test add one auto created queue dynamically and manually modify
-      // capacity
-      ManagedParentQueue parentQueue = (ManagedParentQueue) newCS.getQueue("c");
-      AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1",
-          parentQueue);
-      newCS.addQueue(c1);
-      c1.setEntitlement(new QueueEntitlement(C1_CAPACITY / 100, 1f));
-
-      // Test add another auto created queue and use setEntitlement to modify
-      // capacity
-      AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2",
-          (ManagedParentQueue) newCS.getQueue("c"));
-      newCS.addQueue(c2);
-      newCS.setEntitlement("c2", new QueueEntitlement(C2_CAPACITY / 100, 1f));
-
-      // Verify all allocations match
-      checkQueueCapacities(newCS, C_CAPACITY, D_CAPACITY);
-
-      // Reinitialize and verify all dynamic queued survived
-
-      conf.setCapacity(A, 20f);
-      conf.setCapacity(B, 20f);
-      conf.setCapacity(C, 40f);
-      conf.setCapacity(D, 20f);
-      newCS.reinitialize(conf, newMockRM.getRMContext());
-
-      checkQueueCapacities(newCS, 40f, 20f);
-
-      //chnage parent template configs and reinitialize
-      conf.setAutoCreatedLeafQueueTemplateCapacity(C, 30.0f);
-      conf.setAutoCreatedLeafQueueTemplateMaxCapacity(C, 100.0f);
-      newCS.reinitialize(conf, newMockRM.getRMContext());
+      AutoCreatedLeafQueue user0QueueReinited =
+          (AutoCreatedLeafQueue) cs.getQueue(USER0);
 
-      ManagedParentQueue c = (ManagedParentQueue) newCS.getQueue("c");
-      AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3", c);
-      newCS.addQueue(c3);
+      validateCapacities(user0QueueReinited, 0.0f, 0.0f, 1.0f, 1.0f);
 
-      AbstractManagedParentQueue.AutoCreatedLeafQueueTemplate
-          leafQueueTemplate = parentQueue.getLeafQueueTemplate();
-      QueueCapacities cap = leafQueueTemplate.getQueueCapacities();
-      c3.setEntitlement(
-          new QueueEntitlement(cap.getCapacity(), cap.getMaximumCapacity()));
-      newCS.reinitialize(conf, newMockRM.getRMContext());
+      AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
+          USER1);
+      validateInitialQueueEntitlement(parentQueue, leafQueue.getQueueName(),
+          0.1f);
 
-      checkQueueCapacities(newCS, 40f, 20f);
     } finally {
-      if (newMockRM != null) {
-        ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
-        newMockRM.stop();
-      }
+      cleanupQueue(USER0);
     }
   }
 
@@ -460,7 +234,7 @@ public class TestCapacitySchedulerAutoQueueCreation {
       CapacitySchedulerConfiguration newConf =
           new CapacitySchedulerConfiguration();
       setupQueueConfiguration(newConf);
-      newConf.setAutoCreatedLeafQueueTemplateCapacity(A1, A1_CAPACITY / 10);
+      newConf.setAutoCreatedLeafQueueConfigCapacity(A1, A1_CAPACITY / 10);
       newConf.setAutoCreateChildQueueEnabled(A1, true);
 
       newCS.setConf(new YarnConfiguration());
@@ -490,7 +264,7 @@ public class TestCapacitySchedulerAutoQueueCreation {
       CapacitySchedulerConfiguration newConf =
           new CapacitySchedulerConfiguration();
       setupQueueConfiguration(newConf);
-      newConf.setAutoCreatedLeafQueueTemplateCapacity(A, A_CAPACITY / 10);
+      newConf.setAutoCreatedLeafQueueConfigCapacity(A, A_CAPACITY / 10);
       newConf.setAutoCreateChildQueueEnabled(A, true);
 
       newCS.setConf(new YarnConfiguration());
@@ -531,39 +305,6 @@ public class TestCapacitySchedulerAutoQueueCreation {
     assertEquals(RMAppState.FAILED, app.getState());
   }
 
-  private void validateCapacities(AutoCreatedLeafQueue autoCreatedLeafQueue) {
-    assertEquals(autoCreatedLeafQueue.getCapacity(), 0.0f, EPSILON);
-    assertEquals(autoCreatedLeafQueue.getAbsoluteCapacity(), 0.0f, EPSILON);
-    assertEquals(autoCreatedLeafQueue.getMaximumCapacity(), 0.0f, EPSILON);
-    assertEquals(autoCreatedLeafQueue.getAbsoluteMaximumCapacity(), 0.0f,
-        EPSILON);
-    int maxAppsForAutoCreatedQueues = (int) (
-        CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
-            * autoCreatedLeafQueue.getParent().getAbsoluteCapacity());
-    assertEquals(autoCreatedLeafQueue.getMaxApplicationsPerUser(),
-        maxAppsForAutoCreatedQueues);
-    assertEquals(autoCreatedLeafQueue.getMaxApplicationsPerUser(),
-        (int) (maxAppsForAutoCreatedQueues * (cs.getConfiguration()
-            .getUserLimitFactor(
-                autoCreatedLeafQueue.getParent().getQueuePath()))));
-  }
-
-  private void cleanupQueue(String queueName) throws YarnException {
-    AutoCreatedLeafQueue queue = (AutoCreatedLeafQueue) cs.getQueue(queueName);
-    if (queue != null) {
-      queue.setEntitlement(new QueueEntitlement(0.0f, 0.0f));
-      ((ManagedParentQueue) queue.getParent()).removeChildQueue(
-          queue.getQueueName());
-      cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueueName());
-    } else{
-      throw new YarnException("Queue does not exist " + queueName);
-    }
-  }
-
-  String getQueueMapping(String parentQueue, String leafQueue) {
-    return parentQueue + DOT + leafQueue;
-  }
-
   @Test(timeout = 10000)
   public void testQueueMappingValidationFailsWithInvalidParentQueueInMapping()
       throws Exception {
@@ -586,8 +327,7 @@ public class TestCapacitySchedulerAutoQueueCreation {
         //expected exception
         assertTrue(e.getMessage().contains(
             "invalid parent queue which does not have auto creation of leaf "
-                + "queues enabled ["
-                + "a" + "]"));
+                + "queues enabled [" + "a" + "]"));
       }
 
       //"a" is not auto create enabled and app_user does not exist as a leaf
@@ -650,9 +390,6 @@ public class TestCapacitySchedulerAutoQueueCreation {
         (CapacityScheduler) newMockRM.getResourceScheduler();
 
     try {
-      newMockRM.start();
-      newCS.start();
-
       submitApp(newCS, USER0, USER0, PARENT_QUEUE);
 
       assertNotNull(newCS.getQueue(USER0));
@@ -700,12 +437,16 @@ public class TestCapacitySchedulerAutoQueueCreation {
       AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1",
           parentQueue);
       newCS.addQueue(c1);
-      c1.setEntitlement(new QueueEntitlement(0.5f, 1f));
+      c1.setCapacity(0.5f);
+      c1.setAbsoluteCapacity(c1.getParent().getAbsoluteCapacity() * 1f);
+      c1.setMaxCapacity(1f);
+
+      setEntitlement(c1, new QueueEntitlement(0.5f, 1f));
 
       AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2",
           parentQueue);
       newCS.addQueue(c2);
-      c2.setEntitlement(new QueueEntitlement(0.5f, 1f));
+      setEntitlement(c2, new QueueEntitlement(0.5f, 1f));
 
       try {
         AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3",
@@ -723,72 +464,160 @@ public class TestCapacitySchedulerAutoQueueCreation {
     }
   }
 
-  private List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping(
-      CapacityScheduler newCS, String user, String parentQueue, String queue) {
-    List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
-        new ArrayList<>();
-    queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping(
-        UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user,
-        getQueueMapping(parentQueue, queue)));
-    newCS.getConfiguration().setQueueMappings(queueMappings);
-    return queueMappings;
-  }
+  @Test
+  public void testAutoCreatedQueueActivationDeactivation() throws Exception {
+
+    try {
+      String host = "127.0.0.1";
+      RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
+          host);
+      cs.handle(new NodeAddedSchedulerEvent(node));
+
+      CSQueue parentQueue = cs.getQueue(PARENT_QUEUE);
+
+      //submit app1 as USER1
+      submitApp(mockRM, parentQueue, USER1, USER1, 1, 1);
+      validateInitialQueueEntitlement(parentQueue, USER1, 0.1f);
+
+      //submit another app2 as USER2
+      ApplicationId user2AppId = submitApp(mockRM, parentQueue, USER2, USER2, 2,
+          1);
+      validateInitialQueueEntitlement(parentQueue, USER2, 0.2f);
+
+      //submit another app3 as USER1
+      submitApp(mockRM, parentQueue, USER1, USER1, 3, 2);
+
+      //validate total activated abs capacity remains the same
+      GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
+          (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
+              parentQueue)
+              .getAutoCreatedQueueManagementPolicy();
+      assertEquals(autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), 0.2f, EPSILON);
+
+      //submit user_3 app. This cant be scheduled since there is no capacity
+      submitApp(mockRM, parentQueue, USER3, USER3, 4, 1);
+      final CSQueue user3LeafQueue = cs.getQueue(USER3);
+      validateCapacities((AutoCreatedLeafQueue) user3LeafQueue, 0.0f, 0.0f,
+          1.0f, 1.0f);
 
-  private MockRM setupSchedulerInstance() {
-    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
-    setupQueueConfiguration(conf);
-    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
-        ResourceScheduler.class);
+      assertEquals(autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), 0.2f, EPSILON);
 
-    List<String> queuePlacementRules = new ArrayList<String>();
-    queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
-    conf.setQueuePlacementRules(queuePlacementRules);
+      //deactivate USER2 queue
+      cs.killAllAppsInQueue(USER2);
+      mockRM.waitForState(user2AppId, RMAppState.KILLED);
 
-    setupQueueMappings(conf);
+      //Verify if USER_2 can be deactivated since it has no pending appsA
+      List<QueueManagementChange> queueManagementChanges =
+          autoCreatedQueueManagementPolicy.computeQueueManagementChanges();
 
-    MockRM newMockRM = new MockRM(conf);
-    return newMockRM;
+      ManagedParentQueue managedParentQueue = (ManagedParentQueue) parentQueue;
+      managedParentQueue.validateAndApplyQueueManagementChanges(
+          queueManagementChanges);
+
+      validateDeactivatedQueueEntitlement(parentQueue, USER2, 0.2f,
+          queueManagementChanges);
+
+      //USER_3 should now get activated
+      validateActivatedQueueEntitlement(parentQueue, USER3, 0.2f,
+          queueManagementChanges);
+
+    } finally {
+      cleanupQueue(USER1);
+      cleanupQueue(USER2);
+      cleanupQueue(USER3);
+    }
   }
 
-  void checkQueueCapacities(CapacityScheduler newCS, float capacityC,
-      float capacityD) {
-    CSQueue rootQueue = newCS.getRootQueue();
-    CSQueue queueC = tcs.findQueue(rootQueue, C);
-    CSQueue queueD = tcs.findQueue(rootQueue, D);
-    CSQueue queueC1 = tcs.findQueue(queueC, C1);
-    CSQueue queueC2 = tcs.findQueue(queueC, C2);
-    CSQueue queueC3 = tcs.findQueue(queueC, C3);
-
-    float capC = capacityC / 100.0f;
-    float capD = capacityD / 100.0f;
-
-    tcs.checkQueueCapacity(queueC, capC, capC, 1.0f, 1.0f);
-    tcs.checkQueueCapacity(queueD, capD, capD, 1.0f, 1.0f);
-    tcs.checkQueueCapacity(queueC1, C1_CAPACITY / 100.0f,
-        (C1_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
-    tcs.checkQueueCapacity(queueC2, C2_CAPACITY / 100.0f,
-        (C2_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
-
-    if (queueC3 != null) {
-      ManagedParentQueue parentQueue = (ManagedParentQueue) queueC;
-      QueueCapacities cap =
-          parentQueue.getLeafQueueTemplate().getQueueCapacities();
-      tcs.checkQueueCapacity(queueC3, cap.getCapacity(),
-          (cap.getCapacity()) * capC, 1.0f, 1.0f);
+  @Test
+  public void testAutoCreatedQueueInheritsNodeLabels() throws Exception {
+
+    try {
+      String host = "127.0.0.1";
+      RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
+          host);
+      cs.handle(new NodeAddedSchedulerEvent(node));
+
+      CSQueue parentQueue = cs.getQueue(PARENT_QUEUE);
+
+      submitApp(USER1, USER1, NODEL_LABEL_GPU);
+      //submit app1 as USER1
+      validateInitialQueueEntitlement(parentQueue, USER1, 0.1f);
+    } finally {
+      cleanupQueue(USER1);
     }
   }
 
-  ApplicationAttemptId submitApp(CapacityScheduler newCS, String user,
-      String queue, String parentQueue) {
-    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
-    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, queue, user,
-        new ApplicationPlacementContext(queue, parentQueue));
-    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
-        appId, 1);
-    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(
-        appAttemptId, false);
-    newCS.handle(addAppEvent);
-    newCS.handle(addAttemptEvent);
-    return appAttemptId;
+  @Test
+  public void testReinitializeQueuesWithAutoCreatedLeafQueues()
+      throws Exception {
+
+    MockRM newMockRM = setupSchedulerInstance();
+    try {
+      CapacityScheduler newCS =
+          (CapacityScheduler) newMockRM.getResourceScheduler();
+      CapacitySchedulerConfiguration conf = newCS.getConfiguration();
+
+      String host = "127.0.0.1";
+      RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
+          host);
+      newCS.handle(new NodeAddedSchedulerEvent(node));
+
+      CSQueue parentQueue = newCS.getQueue(PARENT_QUEUE);
+
+      //submit app1 as USER1
+      submitApp(newMockRM, parentQueue, USER1, USER1, 1, 1);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER1, 0.1f);
+
+      //submit another app2 as USER2
+      ApplicationId user2AppId = submitApp(newMockRM, parentQueue, USER2, USER2,
+          2, 1);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER2, 0.2f);
+
+      //update parent queue capacity
+      conf.setCapacity(C, 30f);
+      conf.setCapacity(D, 10f);
+      conf.setMaximumCapacity(C, 50f);
+
+      newCS.reinitialize(conf, newMockRM.getRMContext());
+
+      // validate that leaf queues abs capacity is now changed
+      AutoCreatedLeafQueue user0Queue = (AutoCreatedLeafQueue) newCS.getQueue(
+          USER1);
+      validateCapacities(user0Queue, 0.5f, 0.15f, 1.0f, 0.5f);
+      validateUserAndAppLimits(user0Queue, 1500, 1500);
+
+      //update leaf queue template capacities
+      conf.setAutoCreatedLeafQueueConfigCapacity(C, 30f);
+      conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 40f);
+
+      newCS.reinitialize(conf, newMockRM.getRMContext());
+      validateCapacities(user0Queue, 0.3f, 0.09f, 0.4f, 0.2f);
+      validateUserAndAppLimits(user0Queue, 900, 900);
+
+      //submit app1 as USER3
+      submitApp(newMockRM, parentQueue, USER3, USER3, 3, 1);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER3, 0.27f);
+      AutoCreatedLeafQueue user3Queue = (AutoCreatedLeafQueue) newCS.getQueue(
+          USER1);
+      validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f, 0.2f);
+      validateUserAndAppLimits(user3Queue, 900, 900);
+
+      //submit app1 as USER1 - is already activated. there should be no diff
+      // in capacities
+      submitApp(newMockRM, parentQueue, USER3, USER3, 4, 2);
+      validateInitialQueueEntitlement(newCS, parentQueue, USER3, 0.27f);
+      validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f, 0.2f);
+      validateUserAndAppLimits(user3Queue, 900, 900);
+
+    } finally {
+      cleanupQueue(USER1);
+      cleanupQueue(USER2);
+      if (newMockRM != null) {
+        ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+        newMockRM.stop();
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index 9425d5e..9aba30c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -77,21 +77,21 @@ public class TestCapacitySchedulerDynamicBehavior {
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
 
     //set default queue capacity to zero
-    ((AutoCreatedLeafQueue) cs
+    ((ReservationQueue) cs
             .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
             .setEntitlement(
                     new QueueEntitlement(0f, 1f));
 
     // Test add one reservation dynamically and manually modify capacity
-    AutoCreatedLeafQueue a1 =
-        new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+    ReservationQueue a1 =
+        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
 
     // Test add another reservation queue and use setEntitlement to modify
     // capacity
-    AutoCreatedLeafQueue a2 =
-        new AutoCreatedLeafQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+    ReservationQueue a2 =
+        new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a2);
     cs.setEntitlement("a2", new QueueEntitlement(A2_CAPACITY / 100, 1.0f));
 
@@ -113,8 +113,8 @@ public class TestCapacitySchedulerDynamicBehavior {
 
     try {
       // Test invalid addition (adding non-zero size queue)
-      AutoCreatedLeafQueue a1 =
-          new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+      ReservationQueue a1 =
+          new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
       a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
       cs.addQueue(a1);
       fail();
@@ -123,11 +123,11 @@ public class TestCapacitySchedulerDynamicBehavior {
     }
 
     // Test add one reservation dynamically and manually modify capacity
-    AutoCreatedLeafQueue a1 =
-        new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+    ReservationQueue a1 =
+        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     //set default queue capacity to zero
-    ((AutoCreatedLeafQueue) cs
+    ((ReservationQueue) cs
         .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
             .setEntitlement(
                 new QueueEntitlement(0f, 1f));
@@ -135,8 +135,8 @@ public class TestCapacitySchedulerDynamicBehavior {
 
     // Test add another reservation queue and use setEntitlement to modify
     // capacity
-    AutoCreatedLeafQueue a2 =
-        new AutoCreatedLeafQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
+    ReservationQueue a2 =
+        new ReservationQueue(cs, "a2", (PlanQueue) cs.getQueue("a"));
 
     cs.addQueue(a2);
 
@@ -162,8 +162,8 @@ public class TestCapacitySchedulerDynamicBehavior {
     CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
 
     // Test add one reservation dynamically and manually modify capacity
-    AutoCreatedLeafQueue a1 =
-        new AutoCreatedLeafQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
+    ReservationQueue a1 =
+        new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
     cs.addQueue(a1);
     a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
 
@@ -230,8 +230,8 @@ public class TestCapacitySchedulerDynamicBehavior {
 
     // create the default reservation queue
     String defQName = "a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
-    AutoCreatedLeafQueue defQ =
-        new AutoCreatedLeafQueue(scheduler, defQName,
+    ReservationQueue defQ =
+        new ReservationQueue(scheduler, defQName,
             (PlanQueue) scheduler.getQueue("a"));
     scheduler.addQueue(defQ);
     defQ.setEntitlement(new QueueEntitlement(1f, 1f));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java
new file mode 100644
index 0000000..c71d2bf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestGuaranteedOrZeroCapacityOverTimePolicy.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestGuaranteedOrZeroCapacityOverTimePolicy {
+
+  @Test
+  public void testGetMaxLeavesToBeActivated()
+      throws SchedulerDynamicEditException {
+    GuaranteedOrZeroCapacityOverTimePolicy policy =
+        new GuaranteedOrZeroCapacityOverTimePolicy();
+
+    assertEquals(1, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 1));
+    assertEquals(5, policy.getMaxLeavesToBeActivated(0.17f, 0.03f, 7));
+    assertEquals(0, policy.getMaxLeavesToBeActivated(0, 0.03f, 10));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 1426e88..c45bdb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -18,6 +18,14 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
+    .NO_LABEL;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CSQueueUtils.EPSILON;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CapacitySchedulerConfiguration.ROOT;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -44,6 +52,7 @@ import java.util.concurrent.CyclicBarrier;
 import com.google.common.collect.ImmutableMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -189,7 +198,7 @@ public class TestLeafQueue {
 
     root = 
         CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
-            CapacitySchedulerConfiguration.ROOT, 
+            ROOT,
             queues, queues, 
             TestUtils.spyHook);
     root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
@@ -222,12 +231,12 @@ public class TestLeafQueue {
       final String newRoot) {
     
     // Define top-level queues
-    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot});
-    conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
-    conf.setAcl(CapacitySchedulerConfiguration.ROOT,
+    conf.setQueues(ROOT, new String[] {newRoot});
+    conf.setMaximumCapacity(ROOT, 100);
+    conf.setAcl(ROOT,
       QueueACL.SUBMIT_APPLICATIONS, " ");
     
-    final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot;
+    final String Q_newRoot = ROOT + "." + newRoot;
     conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E});
     conf.setCapacity(Q_newRoot, 100);
     conf.setMaximumCapacity(Q_newRoot, 100);
@@ -410,7 +419,7 @@ public class TestLeafQueue {
     CapacitySchedulerConfiguration testConf = 
         new CapacitySchedulerConfiguration();
     
-    String tproot = CapacitySchedulerConfiguration.ROOT + "." + 
+    String tproot = ROOT + "." +
       "testPolicyRoot" + System.currentTimeMillis();
 
     OrderingPolicy<FiCaSchedulerApp> comPol =    
@@ -485,7 +494,7 @@ public class TestLeafQueue {
     CapacitySchedulerConfiguration testConf =
         new CapacitySchedulerConfiguration();
 
-    String tproot = CapacitySchedulerConfiguration.ROOT + "." +
+    String tproot = ROOT + "." +
       "testPolicyRoot" + System.currentTimeMillis();
 
     OrderingPolicy<FiCaSchedulerApp> schedOrder =
@@ -722,12 +731,12 @@ public class TestLeafQueue {
     Priority priority = TestUtils.createMockPriority(1);
     app0.updateResourceRequests(Collections.singletonList(TestUtils
         .createResourceRequest(ResourceRequest.ANY, 20 * GB, 29, 1, true,
-            priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+            priority, recordFactory, NO_LABEL)));
     assign = b.assignContainers(clusterResource, node0, new ResourceLimits(
         clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
     app0.updateResourceRequests(Collections.singletonList(TestUtils
         .createResourceRequest(ResourceRequest.ANY, 10 * GB, 29, 2, true,
-            priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+            priority, recordFactory, NO_LABEL)));
     assign = b.assignContainers(clusterResource, node0, new ResourceLimits(
         clusterResource), SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
     assertTrue("Still within limits, should assign",
@@ -796,11 +805,11 @@ public class TestLeafQueue {
     Priority priority = TestUtils.createMockPriority(1);
     app0.updateResourceRequests(Collections.singletonList(TestUtils
         .createResourceRequest(ResourceRequest.ANY, 1 * GB, 40, 10, true,
-            priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+            priority, recordFactory, NO_LABEL)));
 
     app2.updateResourceRequests(Collections.singletonList(TestUtils
         .createResourceRequest(ResourceRequest.ANY, 2 * GB, 10, 10, true,
-            priority, recordFactory, RMNodeLabelsManager.NO_LABEL)));
+            priority, recordFactory, NO_LABEL)));
 
     /**
      * Start testing...
@@ -2277,7 +2286,7 @@ public class TestLeafQueue {
         CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, 1);
     Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
     CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
-        csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+        csConf, null, ROOT, newQueues, queues,
         TestUtils.spyHook);
     root.reinitialize(newRoot, cs.getClusterResource());
 
@@ -2712,9 +2721,12 @@ public class TestLeafQueue {
         CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
             * 2);
     Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
-    CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
-        csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
-        TestUtils.spyHook);
+    CSQueue newRoot =
+        CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+            ROOT,
+            newQueues, queues,
+            TestUtils.spyHook);
+    queues = newQueues;
     root.reinitialize(newRoot, csContext.getClusterResource());
 
     // after reinitialization
@@ -2738,7 +2750,7 @@ public class TestLeafQueue {
     Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
     CSQueue newRoot =
         CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
-            CapacitySchedulerConfiguration.ROOT,
+            ROOT,
             newQueues, queues,
             TestUtils.spyHook);
     root.reinitialize(newRoot, cs.getClusterResource());
@@ -3621,7 +3633,7 @@ public class TestLeafQueue {
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(0 * GB,
         e.getTotalPendingResourcesConsideringUserLimit(clusterResource,
-            RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+            NO_LABEL, false).getMemorySize());
 
     // Assign 2nd container of 1GB
     applyCSAssignment(clusterResource,
@@ -3635,7 +3647,7 @@ public class TestLeafQueue {
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
 
     // Can't allocate 3rd container due to user-limit. Headroom still 0.
     applyCSAssignment(clusterResource,
@@ -3645,7 +3657,7 @@ public class TestLeafQueue {
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
 
     // Increase user-limit-factor from 1GB to 10GB (1% * 10 * 100GB = 10GB).
     // Pending for both app_0 and app_1 are still 3GB, so user-limit-factor
@@ -3653,7 +3665,7 @@ public class TestLeafQueue {
     // getTotalPendingResourcesConsideringUserLimit()
     e.setUserLimitFactor(10.0f);
     assertEquals(3*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
 
     applyCSAssignment(clusterResource,
         e.assignContainers(clusterResource, node_0,
@@ -3663,7 +3675,7 @@ public class TestLeafQueue {
     assertEquals(3*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(2*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
 
     // Get the last 2 containers for app_1, no more pending requests.
     applyCSAssignment(clusterResource,
@@ -3677,7 +3689,7 @@ public class TestLeafQueue {
     assertEquals(3*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(2*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
 
     // Release each container from app_0
     for (RMContainer rmContainer : app_0.getLiveContainers()) {
@@ -3788,7 +3800,7 @@ public class TestLeafQueue {
     // With queue capacity set at 1% of 100GB and user-limit-factor set to 1.0,
     // queue 'e' should be able to consume 1GB per user.
     assertEquals(2*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // None of the apps have assigned resources
     // user_0's apps:
     assertEquals(0*GB, app_0.getCurrentConsumption().getMemorySize());
@@ -3805,7 +3817,7 @@ public class TestLeafQueue {
    // The first container was assigned to user_0's app_0. Queues total headroom
     // has 1GB left for user_1.
     assertEquals(1*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
@@ -3823,7 +3835,7 @@ public class TestLeafQueue {
     // this container went to user_0's app_1. so, headroom for queue 'e'e is
     // still 1GB for user_1
     assertEquals(1*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
@@ -3839,7 +3851,7 @@ public class TestLeafQueue {
     // Container was allocated to user_1's app_2 since user_1, Now, no headroom
     // is left.
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
@@ -3855,7 +3867,7 @@ public class TestLeafQueue {
     // Allocated to user_1's app_2 since scheduler allocates 1 container
     // above user resource limit. Available headroom still 0.
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     long app_0_consumption = app_0.getCurrentConsumption().getMemorySize();
     assertEquals(1*GB, app_0_consumption);
@@ -3875,7 +3887,7 @@ public class TestLeafQueue {
     // Cannot allocate 5th container because both users are above their allowed
     // user resource limit. Values should be the same as previously.
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     assertEquals(app_0_consumption, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(app_1_consumption, app_1.getCurrentConsumption().getMemorySize());
@@ -3894,7 +3906,7 @@ public class TestLeafQueue {
         SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), e, nodes, apps);
     // Next container goes to user_0's app_1, since it still wanted 1GB.
     assertEquals(1*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     // user_0's apps:
     assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(2*GB, app_1.getCurrentConsumption().getMemorySize());
@@ -3909,7 +3921,7 @@ public class TestLeafQueue {
     // Last container goes to user_1's app_3, since it still wanted 1GB.
     // user_0's apps:
     assertEquals(0*GB, e.getTotalPendingResourcesConsideringUserLimit(
-        clusterResource, RMNodeLabelsManager.NO_LABEL, false).getMemorySize());
+        clusterResource, NO_LABEL, false).getMemorySize());
     assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(2*GB, app_1.getCurrentConsumption().getMemorySize());
     // user_1's apps:
@@ -4027,6 +4039,59 @@ public class TestLeafQueue {
         app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
   }
 
+  @Test
+  public void testSetupQueueConfigsWithSpecifiedConfiguration()
+      throws IOException {
+
+    try {
+      CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
+          new Configuration(false), false);
+
+      final String leafQueueName =
+          "testSetupQueueConfigsWithSpecifiedConfiguration";
+
+      assertEquals(0, conf.size());
+      conf.setNodeLocalityDelay(60);
+      conf.setCapacity(ROOT + DOT + leafQueueName, 10);
+      conf.setMaximumCapacity(ROOT + DOT + leafQueueName, 100);
+      conf.setUserLimitFactor(ROOT + DOT +leafQueueName, 0.1f);
+
+      csConf.setNodeLocalityDelay(30);
+      csConf.setGlobalMaximumApplicationsPerQueue(20);
+
+      LeafQueue leafQueue = new LeafQueue(csContext, conf,
+          leafQueueName, cs.getRootQueue(),
+          null);
+
+      assertEquals(30, leafQueue.getNodeLocalityDelay());
+      assertEquals(20, leafQueue.getMaxApplications());
+      assertEquals(2, leafQueue.getMaxApplicationsPerUser());
+
+      //check queue configs
+      conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueueName(),
+          NO_LABEL, 10);
+      conf.setMaximumCapacity(leafQueue.getQueueName(), 10);
+
+      assertEquals(0.1, leafQueue.getMaxAMResourcePerQueuePercent(),
+          EPSILON);
+      assertEquals(1, leafQueue.getMaximumCapacity(),
+          EPSILON);
+      assertEquals(0.1, leafQueue.getCapacity(),
+          EPSILON);
+      assertEquals(0.1, leafQueue.getAbsoluteCapacity(),
+          EPSILON);
+      assertEquals(1.0, leafQueue.getAbsoluteMaximumCapacity(),
+          EPSILON);
+
+     } finally {
+        //revert config changes
+        csConf.setNodeLocalityDelay(
+            CapacitySchedulerConfiguration.DEFAULT_NODE_LOCALITY_DELAY);
+        csConf.setGlobalMaximumApplicationsPerQueue(
+            (int) CapacitySchedulerConfiguration.UNDEFINED);
+     }
+  }
+
   private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
     ApplicationId appIdImpl = ApplicationId.newInstance(0, appId);
     ApplicationAttemptId attId =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueManagementDynamicEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueManagementDynamicEditPolicy.java
new file mode 100644
index 0000000..4dc56fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueManagementDynamicEditPolicy.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .queuemanagement.GuaranteedOrZeroCapacityOverTimePolicy;
+import org.junit.Before;
+import org.junit.Test;
+
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CSQueueUtils.EPSILON;
+import static org.junit.Assert.assertEquals;
+
+public class TestQueueManagementDynamicEditPolicy extends
+    TestCapacitySchedulerAutoCreatedQueueBase {
+   private QueueManagementDynamicEditPolicy policy = new
+       QueueManagementDynamicEditPolicy();
+
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+    policy.init(cs.getConfiguration(), cs.getRMContext(), cs);
+  }
+
+  @Test
+  public void testEditSchedule() throws Exception {
+
+    try {
+      policy.editSchedule();
+      assertEquals(2, policy.getManagedParentQueues().size());
+
+      CSQueue parentQueue = cs.getQueue(PARENT_QUEUE);
+
+      GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
+          (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
+              parentQueue)
+              .getAutoCreatedQueueManagementPolicy();
+      assertEquals(0f, autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), EPSILON);
+
+      //submit app1 as USER1
+      ApplicationId user1AppId = submitApp(mockRM, parentQueue, USER1, USER1, 1,
+          1);
+      validateInitialQueueEntitlement(parentQueue, USER1, 0.1f);
+
+      //submit another app2 as USER2
+      ApplicationId user2AppId = submitApp(mockRM, parentQueue, USER2, USER2, 2,
+          1);
+      validateInitialQueueEntitlement(parentQueue, USER2, 0.2f);
+
+      //validate total activated abs capacity
+      assertEquals(0.2f, autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), EPSILON);
+
+      //submit user_3 app. This cant be scheduled since there is no capacity
+      submitApp(mockRM, parentQueue, USER3, USER3, 3, 1);
+      final CSQueue user3LeafQueue = cs.getQueue(USER3);
+      validateCapacities((AutoCreatedLeafQueue) user3LeafQueue, 0.0f, 0.0f,
+          1.0f, 1.0f);
+
+      assertEquals(autoCreatedQueueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity(), 0.2f, EPSILON);
+
+      //deactivate USER2 queue
+      cs.killAllAppsInQueue(USER2);
+      mockRM.waitForState(user2AppId, RMAppState.KILLED);
+
+      //deactivate USER1 queue
+      cs.killAllAppsInQueue(USER1);
+      mockRM.waitForState(user1AppId, RMAppState.KILLED);
+
+      policy.editSchedule();
+
+      waitForPolicyState(0.1f, autoCreatedQueueManagementPolicy, 1000);
+
+      validateCapacities((AutoCreatedLeafQueue) user3LeafQueue, 0.5f, 0.1f,
+          1.0f, 1.0f);
+
+      validateCapacitiesByLabel((ManagedParentQueue) parentQueue, (AutoCreatedLeafQueue) user3LeafQueue,
+          NODEL_LABEL_GPU);
+
+    } finally {
+      cleanupQueue(USER1);
+      cleanupQueue(USER2);
+      cleanupQueue(USER3);
+    }
+  }
+
+  private void waitForPolicyState(float expectedVal,
+      GuaranteedOrZeroCapacityOverTimePolicy queueManagementPolicy, int
+      timesec) throws
+      InterruptedException {
+    long start = System.currentTimeMillis();
+    while (System.currentTimeMillis() - start < timesec * 1000) {
+      if (Float.compare(expectedVal, queueManagementPolicy
+          .getAbsoluteActivatedChildQueueCapacity()) != 0) {
+        Thread.sleep(100);
+      } else {
+        break;
+      }
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[24/50] hadoop git commit: HADOOP-15056. Fix TestUnbuffer#testUnbufferException failure. Contributed by Jack Bearden.

Posted by xy...@apache.org.
HADOOP-15056. Fix TestUnbuffer#testUnbufferException failure. Contributed by Jack Bearden.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19e08942
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19e08942
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19e08942

Branch: refs/heads/HDFS-7240
Commit: 19e089420999dd9d97d981dcd0abd64b6166152d
Parents: 532bbf4
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Dec 7 21:04:05 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Dec 7 21:05:55 2017 -0800

----------------------------------------------------------------------
 .../hadoop/fs/StreamCapabilitiesPolicy.java     | 15 ++++++++++++---
 .../java/org/apache/hadoop/fs/TestUnbuffer.java | 20 +++++++++++++++-----
 2 files changed, 27 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e08942/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
index 3080780..8b63d4b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StreamCapabilitiesPolicy.java
@@ -22,6 +22,8 @@ import java.io.InputStream;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Static methods to implement policies for {@link StreamCapabilities}.
@@ -29,6 +31,10 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class StreamCapabilitiesPolicy {
+  public static final String CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE =
+          "claims unbuffer capabilty but does not implement CanUnbuffer";
+  static final Logger LOG = LoggerFactory.getLogger(
+          StreamCapabilitiesPolicy.class);
   /**
    * Implement the policy for {@link CanUnbuffer#unbuffer()}.
    *
@@ -40,11 +46,14 @@ public class StreamCapabilitiesPolicy {
           && ((StreamCapabilities) in).hasCapability(
           StreamCapabilities.UNBUFFER)) {
         ((CanUnbuffer) in).unbuffer();
+      } else {
+        LOG.debug(in.getClass().getName() + ":"
+                + " does not implement StreamCapabilities"
+                + " and the unbuffer capability");
       }
     } catch (ClassCastException e) {
-      throw new UnsupportedOperationException("this stream " +
-          in.getClass().getName() +
-          " claims to unbuffer but forgets to implement CanUnbuffer");
+      throw new UnsupportedOperationException(in.getClass().getName() + ": "
+              + CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/19e08942/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUnbuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUnbuffer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUnbuffer.java
index b112e30..655d453 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUnbuffer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestUnbuffer.java
@@ -131,17 +131,27 @@ public class TestUnbuffer {
   }
 
   /**
-   * Test unbuffer method which throws an Exception with class name included.
+   * Test that a InputStream should throw an exception when not implementing
+   * CanUnbuffer
+   *
+   * This should throw an exception when the stream claims to have the
+   * unbuffer capability, but actually does not implement CanUnbuffer.
    */
   @Test
   public void testUnbufferException() {
-    FSInputStream in = Mockito.mock(FSInputStream.class);
-    FSDataInputStream fs = new FSDataInputStream(in);
+    abstract class BuggyStream
+            extends FSInputStream
+            implements StreamCapabilities {
+    }
+
+    BuggyStream bs = Mockito.mock(BuggyStream.class);
+    Mockito.when(bs.hasCapability(Mockito.anyString())).thenReturn(true);
 
     exception.expect(UnsupportedOperationException.class);
-    exception.expectMessage("this stream " + in.getClass().getName()
-        + " does not support unbuffering");
+    exception.expectMessage(
+            StreamCapabilitiesPolicy.CAN_UNBUFFER_NOT_IMPLEMENTED_MESSAGE);
 
+    FSDataInputStream fs = new FSDataInputStream(bs);
     fs.unbuffer();
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[06/50] hadoop git commit: YARN-7607. Remove the trailing duplicated timestamp in container diagnostics message. Contributed by Weiwei Yang.

Posted by xy...@apache.org.
YARN-7607. Remove the trailing duplicated timestamp in container diagnostics message. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e411dd66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e411dd66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e411dd66

Branch: refs/heads/HDFS-7240
Commit: e411dd6666041a4ea68ab34e734802271497ae6c
Parents: c2e8a5c
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Dec 7 17:29:40 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Dec 7 17:29:40 2017 +0800

----------------------------------------------------------------------
 .../container/ContainerImpl.java                | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e411dd66/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 901b013..f95c2a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -1541,7 +1541,7 @@ public class ContainerImpl implements Container {
       ContainerExitEvent exitEvent = (ContainerExitEvent) event;
       container.exitCode = exitEvent.getExitCode();
       if (exitEvent.getDiagnosticInfo() != null) {
-        container.addDiagnostics(exitEvent.getDiagnosticInfo(), "\n");
+        container.addDiagnostics(exitEvent.getDiagnosticInfo() + "\n");
       }
 
       // TODO: Add containerWorkDir to the deletion service.
@@ -1578,7 +1578,7 @@ public class ContainerImpl implements Container {
           container.addDiagnostics("Diagnostic message from attempt "
               + n + " : ", "\n");
         }
-        container.addDiagnostics(exitEvent.getDiagnosticInfo(), "\n");
+        container.addDiagnostics(exitEvent.getDiagnosticInfo() + "\n");
       }
 
       if (container.shouldRetry(container.exitCode)) {
@@ -1737,7 +1737,7 @@ public class ContainerImpl implements Container {
 
       ContainerResourceFailedEvent rsrcFailedEvent =
           (ContainerResourceFailedEvent) event;
-      container.addDiagnostics(rsrcFailedEvent.getDiagnosticMessage(), "\n");
+      container.addDiagnostics(rsrcFailedEvent.getDiagnosticMessage() + "\n");
 
       // Inform the localizer to decrement reference counts and cleanup
       // resources.
@@ -1760,7 +1760,7 @@ public class ContainerImpl implements Container {
       container.metrics.endInitingContainer();
       ContainerKillEvent killEvent = (ContainerKillEvent) event;
       container.exitCode = killEvent.getContainerExitStatus();
-      container.addDiagnostics(killEvent.getDiagnostic(), "\n");
+      container.addDiagnostics(killEvent.getDiagnostic() + "\n");
       container.addDiagnostics("Container is killed before being launched.\n");
     }
   }
@@ -1799,7 +1799,7 @@ public class ContainerImpl implements Container {
           new ContainersLauncherEvent(container,
               ContainersLauncherEventType.CLEANUP_CONTAINER));
       ContainerKillEvent killEvent = (ContainerKillEvent) event;
-      container.addDiagnostics(killEvent.getDiagnostic(), "\n");
+      container.addDiagnostics(killEvent.getDiagnostic() + "\n");
       container.exitCode = killEvent.getContainerExitStatus();
     }
   }
@@ -1838,7 +1838,7 @@ public class ContainerImpl implements Container {
       }
 
       if (exitEvent.getDiagnosticInfo() != null) {
-        container.addDiagnostics(exitEvent.getDiagnosticInfo(), "\n");
+        container.addDiagnostics(exitEvent.getDiagnosticInfo() + "\n");
       }
 
       // The process/process-grp is killed. Decrement reference counts and
@@ -1890,7 +1890,7 @@ public class ContainerImpl implements Container {
       } else {
         ContainerKillEvent killEvent = (ContainerKillEvent) event;
         container.exitCode = killEvent.getContainerExitStatus();
-        container.addDiagnostics(killEvent.getDiagnostic(), "\n");
+        container.addDiagnostics(killEvent.getDiagnostic() + "\n");
         container.addDiagnostics("Container is killed before being launched.\n");
         container.metrics.killedContainer();
         NMAuditLogger.logSuccess(container.user,
@@ -2010,7 +2010,7 @@ public class ContainerImpl implements Container {
     public void transition(ContainerImpl container, ContainerEvent event) {
       ContainerDiagnosticsUpdateEvent updateEvent =
           (ContainerDiagnosticsUpdateEvent) event;
-      container.addDiagnostics(updateEvent.getDiagnosticsUpdate(), "\n");
+      container.addDiagnostics(updateEvent.getDiagnosticsUpdate() + "\n");
     }
   }
 
@@ -2028,7 +2028,7 @@ public class ContainerImpl implements Container {
           new ContainersLauncherEvent(container,
               ContainersLauncherEventType.PAUSE_CONTAINER));
       ContainerPauseEvent pauseEvent = (ContainerPauseEvent) event;
-      container.addDiagnostics(pauseEvent.getDiagnostic(), "\n");
+      container.addDiagnostics(pauseEvent.getDiagnostic() + "\n");
     }
   }
 
@@ -2061,7 +2061,7 @@ public class ContainerImpl implements Container {
           new ContainersLauncherEvent(container,
               ContainersLauncherEventType.RESUME_CONTAINER));
       ContainerResumeEvent resumeEvent = (ContainerResumeEvent) event;
-      container.addDiagnostics(resumeEvent.getDiagnostic(), "\n");
+      container.addDiagnostics(resumeEvent.getDiagnostic() + "\n");
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[36/50] hadoop git commit: YARN-7443. Add native FPGA module support to do isolation with cgroups. (Zhankun Tang via wangda)

Posted by xy...@apache.org.
YARN-7443. Add native FPGA module support to do isolation with cgroups. (Zhankun Tang via wangda)

Change-Id: Ic4b7f9f3e032986b8f955139c9fe4d3a6c818a53


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04b84da2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04b84da2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04b84da2

Branch: refs/heads/HDFS-7240
Commit: 04b84da2456fb8c716e728b16db4851e2e87ec25
Parents: adca1a7
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:18:22 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:18:22 2017 -0800

----------------------------------------------------------------------
 .../hadoop-yarn/conf/container-executor.cfg     |   6 +
 .../src/CMakeLists.txt                          |   2 +
 .../main/native/container-executor/impl/main.c  |   6 +
 .../impl/modules/fpga/fpga-module.c             | 229 +++++++++++++++++++
 .../impl/modules/fpga/fpga-module.h             |  45 ++++
 .../test/modules/fpga/test-fpga-module.cc       | 216 +++++++++++++++++
 6 files changed, 504 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
index 7a84d76..4c698b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
+++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg
@@ -15,3 +15,9 @@ feature.tc.enabled=0
 #  docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs
 #  docker.privileged-containers.enabled=0
 #  docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers
+
+# The configs below deal with settings for FPGA resource
+#[fpga]
+#  module.enabled=## Enable/Disable the FPGA resource handler module. set to "true" to enable, disabled by default
+#  fpga.major-device-number=## Major device number of FPGA, by default is 246. Strongly recommend setting this
+#  fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by YARN.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 9d83bf3..79faeec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -132,6 +132,7 @@ add_library(container
     main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
     main/native/container-executor/impl/modules/common/module-configs.c
     main/native/container-executor/impl/modules/gpu/gpu-module.c
+    main/native/container-executor/impl/modules/fpga/fpga-module.c
     main/native/container-executor/impl/utils/docker-util.c
 )
 
@@ -165,6 +166,7 @@ add_executable(cetest
         main/native/container-executor/test/utils/test-path-utils.cc
         main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
         main/native/container-executor/test/modules/gpu/test-gpu-module.cc
+        main/native/container-executor/test/modules/fpga/test-fpga-module.cc
         main/native/container-executor/test/test_util.cc
         main/native/container-executor/test/utils/test_docker_util.cc)
 target_link_libraries(cetest gtest container)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 7b8f63f..ca84d40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -22,6 +22,7 @@
 #include "util.h"
 #include "get_executable.h"
 #include "modules/gpu/gpu-module.h"
+#include "modules/fpga/fpga-module.h"
 #include "modules/cgroups/cgroups-operations.h"
 
 #include <errno.h>
@@ -244,6 +245,11 @@ static int validate_arguments(int argc, char **argv , int *operation) {
            &argv[1]);
   }
 
+  if (strcmp("--module-fpga", argv[1]) == 0) {
+    return handle_fpga_request(&update_cgroups_parameters, "fpga", argc - 1,
+           &argv[1]);
+  }
+
   if (strcmp("--checksetup", argv[1]) == 0) {
     *operation = CHECK_SETUP;
     return 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
new file mode 100644
index 0000000..c1a2f83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.c
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "configuration.h"
+#include "container-executor.h"
+#include "utils/string-utils.h"
+#include "modules/fpga/fpga-module.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "modules/common/module-configs.h"
+#include "modules/common/constants.h"
+#include "util.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <unistd.h>
+
+#define EXCLUDED_FPGAS_OPTION "excluded_fpgas"
+#define CONTAINER_ID_OPTION "container_id"
+#define DEFAULT_INTEL_MAJOR_NUMBER 246
+#define MAX_CONTAINER_ID_LEN 128
+
+static const struct section* cfg_section;
+
+static int internal_handle_fpga_request(
+    update_cgroups_parameters_function update_cgroups_parameters_func_p,
+    size_t n_minor_devices_to_block, int minor_devices[],
+    const char* container_id) {
+  char* allowed_minor_numbers_str = NULL;
+  int* allowed_minor_numbers = NULL;
+  size_t n_allowed_minor_numbers = 0;
+  int return_code = 0;
+
+  if (n_minor_devices_to_block == 0) {
+    // no device to block, just return;
+    return 0;
+  }
+
+  // Get major device number from cfg, if not set, major number of (Intel)
+  // will be the default value.
+  int major_device_number;
+  char* major_number_str = get_section_value(FPGA_MAJOR_NUMBER_CONFIG_KEY,
+     cfg_section);
+  if (!major_number_str || 0 == major_number_str[0]) {
+    // Default major number of Intel devices
+    major_device_number = DEFAULT_INTEL_MAJOR_NUMBER;
+  } else {
+    major_device_number = strtol(major_number_str, NULL, 0);
+  }
+
+  // Get allowed minor device numbers from cfg, if not set, means all minor
+  // devices can be used by YARN
+  allowed_minor_numbers_str = get_section_value(
+      FPGA_ALLOWED_DEVICES_MINOR_NUMBERS,
+      cfg_section);
+  if (!allowed_minor_numbers_str || 0 == allowed_minor_numbers_str[0]) {
+    allowed_minor_numbers = NULL;
+  } else {
+    int rc = get_numbers_split_by_comma(allowed_minor_numbers_str,
+                                        &allowed_minor_numbers,
+                                        &n_allowed_minor_numbers);
+    if (0 != rc) {
+      fprintf(ERRORFILE,
+          "Failed to get allowed minor device numbers from cfg, value=%s\n",
+          allowed_minor_numbers_str);
+      return_code = -1;
+      goto cleanup;
+    }
+
+    // Make sure we're trying to black devices allowed in config
+    for (int i = 0; i < n_minor_devices_to_block; i++) {
+      int found = 0;
+      for (int j = 0; j < n_allowed_minor_numbers; j++) {
+        if (minor_devices[i] == allowed_minor_numbers[j]) {
+          found = 1;
+          break;
+        }
+      }
+
+      if (!found) {
+        fprintf(ERRORFILE,
+          "Trying to blacklist device with minor-number=%d which is not on allowed list\n",
+          minor_devices[i]);
+        return_code = -1;
+        goto cleanup;
+      }
+    }
+  }
+
+  // Use cgroup helpers to blacklist devices
+  for (int i = 0; i < n_minor_devices_to_block; i++) {
+    char param_value[128];
+    memset(param_value, 0, sizeof(param_value));
+    snprintf(param_value, sizeof(param_value), "c %d:%d rwm",
+             major_device_number, minor_devices[i]);
+
+    int rc = update_cgroups_parameters_func_p("devices", "deny",
+      container_id, param_value);
+
+    if (0 != rc) {
+      fprintf(ERRORFILE, "CGroups: Failed to update cgroups\n");
+      return_code = -1;
+      goto cleanup;
+    }
+  }
+
+cleanup:
+  if (major_number_str) {
+    free(major_number_str);
+  }
+  if (allowed_minor_numbers) {
+    free(allowed_minor_numbers);
+  }
+  if (allowed_minor_numbers_str) {
+    free(allowed_minor_numbers_str);
+  }
+
+  return return_code;
+}
+
+void reload_fpga_configuration() {
+  cfg_section = get_configuration_section(FPGA_MODULE_SECTION_NAME, get_cfg());
+}
+
+/*
+ * Format of FPGA request commandline:
+ *
+ * c-e fpga --excluded_fpgas 0,1,3 --container_id container_x_y
+ */
+int handle_fpga_request(update_cgroups_parameters_function func,
+    const char* module_name, int module_argc, char** module_argv) {
+  if (!cfg_section) {
+    reload_fpga_configuration();
+  }
+
+  if (!module_enabled(cfg_section, FPGA_MODULE_SECTION_NAME)) {
+    fprintf(ERRORFILE,
+      "Please make sure fpga module is enabled before using it.\n");
+    return -1;
+  }
+
+  static struct option long_options[] = {
+    {EXCLUDED_FPGAS_OPTION, required_argument, 0, 'e' },
+    {CONTAINER_ID_OPTION, required_argument, 0, 'c' },
+    {0, 0, 0, 0}
+  };
+
+  int rc = 0;
+  int c = 0;
+  int option_index = 0;
+
+  int* minor_devices = NULL;
+  char container_id[MAX_CONTAINER_ID_LEN];
+  memset(container_id, 0, sizeof(container_id));
+  size_t n_minor_devices_to_block = 0;
+  int failed = 0;
+
+  optind = 1;
+  while((c = getopt_long(module_argc, module_argv, "e:c:",
+                         long_options, &option_index)) != -1) {
+    switch(c) {
+      case 'e':
+        rc = get_numbers_split_by_comma(optarg, &minor_devices,
+          &n_minor_devices_to_block);
+        if (0 != rc) {
+          fprintf(ERRORFILE,
+            "Failed to get minor devices number from command line, value=%s\n",
+            optarg);
+          failed = 1;
+          goto cleanup;
+        }
+        break;
+      case 'c':
+        if (!validate_container_id(optarg)) {
+          fprintf(ERRORFILE,
+            "Specified container_id=%s is invalid\n", optarg);
+          failed = 1;
+          goto cleanup;
+        }
+        strncpy(container_id, optarg, MAX_CONTAINER_ID_LEN);
+        break;
+      default:
+        fprintf(ERRORFILE,
+          "Unknown option in fpga command character %d %c, optionindex = %d\n",
+          c, c, optind);
+        failed = 1;
+        goto cleanup;
+    }
+  }
+
+  if (0 == container_id[0]) {
+    fprintf(ERRORFILE,
+      "[%s] --container_id must be specified.\n", __func__);
+    failed = 1;
+    goto cleanup;
+  }
+
+  if (!minor_devices) {
+     // Minor devices is null, skip following call.
+     fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+     goto cleanup;
+  }
+
+  failed = internal_handle_fpga_request(func, n_minor_devices_to_block,
+         minor_devices,
+         container_id);
+
+cleanup:
+  if (minor_devices) {
+    free(minor_devices);
+  }
+  return failed;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.h
new file mode 100644
index 0000000..12d6ab3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/fpga/fpga-module.h
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef __FreeBSD__
+#define _WITH_GETLINE
+#endif
+
+#ifndef _MODULES_FPGA_FPGA_MUDULE_H_
+#define _MODULES_FPGA_FPGA_MUDULE_H_
+
+#define FPGA_MAJOR_NUMBER_CONFIG_KEY "fpga.major-device-number"
+#define FPGA_ALLOWED_DEVICES_MINOR_NUMBERS "fpga.allowed-device-minor-numbers"
+#define FPGA_MODULE_SECTION_NAME "fpga"
+
+// For unit test stubbing
+typedef int (*update_cgroups_parameters_function)(const char*, const char*,
+   const char*, const char*);
+
+/**
+ * Handle fpga requests
+ */
+int handle_fpga_request(update_cgroups_parameters_function func,
+   const char* module_name, int module_argc, char** module_argv);
+
+/**
+ * Reload config from filesystem, visible for testing.
+ */
+void reload_fpga_configuration();
+
+#endif
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04b84da2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
new file mode 100644
index 0000000..1e5c5ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/fpga/test-fpga-module.cc
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vector>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <gtest/gtest.h>
+#include <sstream>
+
+extern "C" {
+#include "configuration.h"
+#include "container-executor.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "modules/fpga/fpga-module.h"
+#include "test/test-container-executor-common.h"
+#include "util.h"
+}
+
+namespace ContainerExecutor {
+
+class TestFpgaModule : public ::testing::Test {
+protected:
+  virtual void SetUp() {
+    if (mkdirs(TEST_ROOT, 0755) != 0) {
+      fprintf(ERRORFILE, "Failed to mkdir TEST_ROOT: %s\n", TEST_ROOT);
+      exit(1);
+    }
+    LOGFILE = stdout;
+    ERRORFILE = stderr;
+  }
+
+  virtual void TearDown() {
+
+  }
+};
+
+static std::vector<const char*> cgroups_parameters_invoked;
+
+static int mock_update_cgroups_parameters(
+   const char* controller_name,
+   const char* param_name,
+   const char* group_id,
+   const char* value) {
+  char* buf = (char*) malloc(128);
+  strcpy(buf, controller_name);
+  cgroups_parameters_invoked.push_back(buf);
+
+  buf = (char*) malloc(128);
+  strcpy(buf, param_name);
+  cgroups_parameters_invoked.push_back(buf);
+
+  buf = (char*) malloc(128);
+  strcpy(buf, group_id);
+  cgroups_parameters_invoked.push_back(buf);
+
+  buf = (char*) malloc(128);
+  strcpy(buf, value);
+  cgroups_parameters_invoked.push_back(buf);
+  return 0;
+}
+
+static void verify_param_updated_to_cgroups(
+    int argc, const char** argv) {
+  ASSERT_EQ(argc, cgroups_parameters_invoked.size());
+
+  int offset = 0;
+  while (offset < argc) {
+    ASSERT_STREQ(argv[offset], cgroups_parameters_invoked[offset]);
+    offset++;
+  }
+}
+
+static void write_and_load_fpga_module_to_cfg(const char* cfg_filepath, int enabled) {
+  FILE *file = fopen(cfg_filepath, "w");
+  if (file == NULL) {
+    printf("FAIL: Could not open configuration file: %s\n", cfg_filepath);
+    exit(1);
+  }
+  fprintf(file, "[fpga]\n");
+  if (enabled) {
+    fprintf(file, "module.enabled=true\n");
+  } else {
+    fprintf(file, "module.enabled=false\n");
+  }
+  fclose(file);
+
+  // Read config file
+  read_executor_config(cfg_filepath);
+  reload_fpga_configuration();
+}
+
+static void test_fpga_module_enabled_disabled(int enabled) {
+  // Write config file.
+  const char *filename = TEST_ROOT "/test_cgroups_module_enabled_disabled.cfg";
+  write_and_load_fpga_module_to_cfg(filename, enabled);
+
+  char* argv[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
+                   (char*) "--container_id",
+                   (char*) "container_1498064906505_0001_01_000001" };
+
+  int rc = handle_fpga_request(&mock_update_cgroups_parameters,
+              "fpga", 5, argv);
+
+  int EXPECTED_RC;
+  if (enabled) {
+    EXPECTED_RC = 0;
+  } else {
+    EXPECTED_RC = -1;
+  }
+  ASSERT_EQ(EXPECTED_RC, rc);
+}
+
+TEST_F(TestFpgaModule, test_verify_fpga_module_calls_cgroup_parameter) {
+  // Write config file.
+  const char *filename = TEST_ROOT "/test_verify_fpga_module_calls_cgroup_parameter.cfg";
+  write_and_load_fpga_module_to_cfg(filename, 1);
+
+  char* container_id = (char*) "container_1498064906505_0001_01_000001";
+  char* argv[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
+                   (char*) "--container_id",
+                   container_id };
+
+  /* Test case 1: block 2 devices */
+  cgroups_parameters_invoked.clear();
+  int rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 5, argv);
+  ASSERT_EQ(0, rc) << "Should success.\n";
+
+  // Verify cgroups parameters
+  const char* expected_cgroups_argv[] = { "devices", "deny", container_id, "c 246:0 rwm",
+    "devices", "deny", container_id, "c 246:1 rwm"};
+  verify_param_updated_to_cgroups(8, expected_cgroups_argv);
+
+  /* Test case 2: block 0 devices */
+  cgroups_parameters_invoked.clear();
+  char* argv_1[] = { (char*) "--module-fpga", (char*) "--container_id", container_id };
+  rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 3, argv_1);
+  ASSERT_EQ(0, rc) << "Should success.\n";
+
+  // Verify cgroups parameters
+  verify_param_updated_to_cgroups(0, NULL);
+
+  /* Test case 3: block 2 non-sequential devices */
+  cgroups_parameters_invoked.clear();
+  char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "1,3",
+                   (char*) "--container_id", container_id };
+  rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 5, argv_2);
+  ASSERT_EQ(0, rc) << "Should success.\n";
+
+  // Verify cgroups parameters
+  const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 246:1 rwm",
+    "devices", "deny", container_id, "c 246:3 rwm"};
+  verify_param_updated_to_cgroups(8, expected_cgroups_argv_2);
+}
+
+TEST_F(TestFpgaModule, test_illegal_cli_parameters) {
+  // Write config file.
+  const char *filename = TEST_ROOT "/test_illegal_cli_parameters.cfg";
+  write_and_load_fpga_module_to_cfg(filename, 1);
+
+  // Illegal container id - 1
+  char* argv[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
+                   (char*) "--container_id", (char*) "xxxx" };
+  int rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 5, argv);
+  ASSERT_NE(0, rc) << "Should fail.\n";
+
+  // Illegal container id - 2
+  char* argv_1[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1",
+                   (char*) "--container_id", (char*) "container_1" };
+  rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 5, argv_1);
+  ASSERT_NE(0, rc) << "Should fail.\n";
+
+  // Illegal container id - 3
+  char* argv_2[] = { (char*) "--module-fpga", (char*) "--excluded_fpgas", (char*) "0,1" };
+  rc = handle_fpga_request(&mock_update_cgroups_parameters,
+     "fpga", 3, argv_2);
+  ASSERT_NE(0, rc) << "Should fail.\n";
+}
+
+TEST_F(TestFpgaModule, test_fpga_module_disabled) {
+  test_fpga_module_enabled_disabled(0);
+}
+
+TEST_F(TestFpgaModule, test_fpga_module_enabled) {
+  test_fpga_module_enabled_disabled(1);
+}
+} // namespace ContainerExecutor
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[49/50] hadoop git commit: YARN-7595. Container launching code suppresses close exceptions after writes. Contributed by Jim Brennan

Posted by xy...@apache.org.
YARN-7595. Container launching code suppresses close exceptions after writes. Contributed by Jim Brennan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2abab1d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2abab1d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2abab1d7

Branch: refs/heads/HDFS-7240
Commit: 2abab1d7c53e64c160384fd5a3ac4cd8ffa57af4
Parents: 3ebe6a7
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Dec 12 16:04:15 2017 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Dec 12 16:04:15 2017 -0600

----------------------------------------------------------------------
 .../nodemanager/DefaultContainerExecutor.java   | 25 +++------
 .../launcher/ContainerLaunch.java               | 54 +++++++++-----------
 .../JavaSandboxLinuxContainerRuntime.java       | 17 +++---
 3 files changed, 41 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2abab1d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 5d78f9d..5772403 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -42,12 +42,10 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.CommandExecutor;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -414,15 +412,11 @@ public class DefaultContainerExecutor extends ContainerExecutor {
      */
     public void writeLocalWrapperScript(Path launchDst, Path pidFile)
         throws IOException {
-      DataOutputStream out = null;
-      PrintStream pout = null;
-
-      try {
-        out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out, false, "UTF-8");
+      try (DataOutputStream out =
+               lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE));
+           PrintStream pout =
+               new PrintStream(out, false, "UTF-8")) {
         writeLocalWrapperScript(launchDst, pidFile, pout);
-      } finally {
-        IOUtils.cleanupWithLogger(LOG, pout, out);
       }
     }
 
@@ -489,11 +483,10 @@ public class DefaultContainerExecutor extends ContainerExecutor {
 
     private void writeSessionScript(Path launchDst, Path pidFile)
         throws IOException {
-      DataOutputStream out = null;
-      PrintStream pout = null;
-      try {
-        out = lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE));
-        pout = new PrintStream(out, false, "UTF-8");
+      try (DataOutputStream out =
+               lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE));
+           PrintStream pout =
+               new PrintStream(out, false, "UTF-8")) {
         // We need to do a move as writing to a file is not atomic
         // Process reading a file being written to may get garbled data
         // hence write pid to tmp file first followed by a mv
@@ -503,8 +496,6 @@ public class DefaultContainerExecutor extends ContainerExecutor {
         pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile);
         String exec = Shell.isSetsidAvailable? "exec setsid" : "exec";
         pout.printf("%s /bin/bash \"%s\"", exec, launchDst.toUri().getPath());
-      } finally {
-        IOUtils.cleanupWithLogger(LOG, pout, out);
       }
       lfs.setPermission(sessionScriptPath,
           ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2abab1d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f1c826e..db90215 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -220,15 +220,13 @@ public class ContainerLaunch implements Callable<Integer> {
               containerIdStr));
       Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(
           getContainerPrivateDir(appIdStr, containerIdStr));
-      DataOutputStream containerScriptOutStream = null;
-      DataOutputStream tokensOutStream = null;
 
       // Select the working directory for the container
       Path containerWorkDir = deriveContainerWorkDir();
       recordContainerWorkDir(containerID, containerWorkDir.toString());
 
       String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
-      // pid file should be in nm private dir so that it is not 
+      // pid file should be in nm private dir so that it is not
       // accessible by users
       pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
       List<String> localDirs = dirsHandler.getLocalDirs();
@@ -243,24 +241,24 @@ public class ContainerLaunch implements Callable<Integer> {
         throw new IOException("Most of the disks failed. "
             + dirsHandler.getDisksHealthReport(false));
       }
-      try {
-        // /////////// Write out the container-script in the nmPrivate space.
-        List<Path> appDirs = new ArrayList<Path>(localDirs.size());
-        for (String localDir : localDirs) {
-          Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
-          Path userdir = new Path(usersdir, user);
-          Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
-          appDirs.add(new Path(appsdir, appIdStr));
-        }
-        containerScriptOutStream =
-            lfs.create(nmPrivateContainerScriptPath,
-                EnumSet.of(CREATE, OVERWRITE));
-
-        // Set the token location too.
-        environment.put(
-            ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
-            new Path(containerWorkDir,
-                FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
+      List<Path> appDirs = new ArrayList<Path>(localDirs.size());
+      for (String localDir : localDirs) {
+        Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
+        Path userdir = new Path(usersdir, user);
+        Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
+        appDirs.add(new Path(appsdir, appIdStr));
+      }
+
+      // Set the token location too.
+      environment.put(
+          ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
+          new Path(containerWorkDir,
+              FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
+
+      // /////////// Write out the container-script in the nmPrivate space.
+      try (DataOutputStream containerScriptOutStream =
+               lfs.create(nmPrivateContainerScriptPath,
+                   EnumSet.of(CREATE, OVERWRITE))) {
         // Sanitize the container's environment
         sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
             containerLogDirs, localResources, nmPrivateClasspathJarDir);
@@ -271,18 +269,16 @@ public class ContainerLaunch implements Callable<Integer> {
         exec.writeLaunchEnv(containerScriptOutStream, environment,
             localResources, launchContext.getCommands(),
             new Path(containerLogDirs.get(0)), user);
-        // /////////// End of writing out container-script
+      }
+      // /////////// End of writing out container-script
 
-        // /////////// Write out the container-tokens in the nmPrivate space.
-        tokensOutStream =
-            lfs.create(nmPrivateTokensPath, EnumSet.of(CREATE, OVERWRITE));
+      // /////////// Write out the container-tokens in the nmPrivate space.
+      try (DataOutputStream tokensOutStream =
+               lfs.create(nmPrivateTokensPath, EnumSet.of(CREATE, OVERWRITE))) {
         Credentials creds = container.getCredentials();
         creds.writeTokenStorageToStream(tokensOutStream);
-        // /////////// End of writing out container-tokens
-      } finally {
-        IOUtils.cleanupWithLogger(LOG, containerScriptOutStream,
-            tokensOutStream);
       }
+      // /////////// End of writing out container-tokens
 
       ret = launchContainer(new ContainerStartContext.Builder()
           .setContainer(container)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2abab1d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 245b38f..1ab1fc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -231,7 +231,6 @@ public class JavaSandboxLinuxContainerRuntime
         throw new ContainerExecutionException("hadoop.tmp.dir not set!");
       }
 
-      OutputStream policyOutputStream = null;
       try {
         String containerID = ctx.getExecutionAttribute(CONTAINER_ID_STR);
         initializePolicyDir();
@@ -242,19 +241,19 @@ public class JavaSandboxLinuxContainerRuntime
             Paths.get(policyFileDir.toString(),
             containerID + "-" + NMContainerPolicyUtils.POLICY_FILE),
             POLICY_ATTR);
-        policyOutputStream = Files.newOutputStream(policyFilePath);
 
-        containerPolicies.put(containerID, policyFilePath);
+        try(OutputStream policyOutputStream =
+                Files.newOutputStream(policyFilePath)) {
 
-        NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
-            localDirs, groupPolicyFiles, resources, configuration);
-        NMContainerPolicyUtils.appendSecurityFlags(
-            commands, env, policyFilePath, sandboxMode);
+          containerPolicies.put(containerID, policyFilePath);
 
+          NMContainerPolicyUtils.generatePolicyFile(policyOutputStream,
+              localDirs, groupPolicyFiles, resources, configuration);
+          NMContainerPolicyUtils.appendSecurityFlags(
+              commands, env, policyFilePath, sandboxMode);
+        }
       } catch (IOException e) {
         throw new ContainerExecutionException(e);
-      } finally {
-        IOUtils.cleanupWithLogger(LOG, policyOutputStream);
       }
     }
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[42/50] hadoop git commit: HDFS-12875. RBF: Complete logic for -readonly option of dfsrouteradmin add command. Contributed by Inigo Goiri.

Posted by xy...@apache.org.
HDFS-12875. RBF: Complete logic for -readonly option of dfsrouteradmin add command. Contributed by Inigo Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5cd1056a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5cd1056a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5cd1056a

Branch: refs/heads/HDFS-7240
Commit: 5cd1056ad77a2ebb0466e7bc5933337b57f6fe30
Parents: 2316f52
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Dec 11 15:14:57 2017 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Dec 11 15:14:57 2017 -0800

----------------------------------------------------------------------
 .../federation/router/RouterRpcServer.java      |  34 +++++
 .../hdfs/tools/federation/RouterAdmin.java      |   2 +-
 .../src/site/markdown/HDFSCommands.md           |   2 +-
 .../src/site/markdown/HDFSRouterFederation.md   |   4 +
 .../server/federation/FederationTestUtils.java  |  60 ++++----
 .../server/federation/RouterDFSCluster.java     |   5 +-
 .../resolver/TestMountTableResolver.java        |  34 +++--
 .../federation/router/TestRouterAdmin.java      |  31 ++++
 .../federation/router/TestRouterMountTable.java | 143 +++++++++++++++++++
 9 files changed, 276 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index b5acf12..d401e69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -103,8 +103,10 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
 import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
 import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
 import org.apache.hadoop.hdfs.server.federation.resolver.PathLocation;
 import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
 import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
@@ -1982,6 +1984,17 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
             this.subclusterResolver);
       }
 
+      // We may block some write operations
+      if (opCategory.get() == OperationCategory.WRITE) {
+        // Check if the path is in a read only mount point
+        if (isPathReadOnly(path)) {
+          if (this.rpcMonitor != null) {
+            this.rpcMonitor.routerFailureReadOnly();
+          }
+          throw new IOException(path + " is in a read only mount point");
+        }
+      }
+
       return location.getDestinations();
     } catch (IOException ioe) {
       if (this.rpcMonitor != null) {
@@ -1992,6 +2005,27 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
   }
 
   /**
+   * Check if a path is in a read only mount point.
+   *
+   * @param path Path to check.
+   * @return If the path is in a read only mount point.
+   */
+  private boolean isPathReadOnly(final String path) {
+    if (subclusterResolver instanceof MountTableResolver) {
+      try {
+        MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
+        MountTable entry = mountTable.getMountPoint(path);
+        if (entry != null && entry.isReadOnly()) {
+          return true;
+        }
+      } catch (IOException e) {
+        LOG.error("Cannot get mount point: {}", e.getMessage());
+      }
+    }
+    return false;
+  }
+
+  /**
    * Get the modification dates for mount points.
    *
    * @param path Name of the path to start checking dates from.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 0786419..897432f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -77,7 +77,7 @@ public class RouterAdmin extends Configured implements Tool {
   public void printUsage() {
     String usage = "Federation Admin Tools:\n"
         + "\t[-add <source> <nameservice> <destination> "
-        + "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL]]\n"
+        + "[-readonly]\n"
         + "\t[-rm <source>]\n"
         + "\t[-ls <path>]\n";
     System.out.println(usage);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index d8462c1..16a47fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -425,7 +425,7 @@ Runs the DFS router. See [Router](./HDFSRouterFederation.html#Router) for more i
 Usage:
 
       hdfs dfsrouteradmin
-          [-add <source> <nameservice> <destination>]
+          [-add <source> <nameservice> <destination> [-readonly]]
           [-rm <source>]
           [-ls <path>]
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
index 2cad0f1..dad231d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md
@@ -184,6 +184,10 @@ For example, to create three mount points and list them:
     [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /data/app2 ns3 /data/app2
     [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -ls
 
+It also supports mount points that disallow writes:
+
+    [hdfs]$ $HADOOP_HOME/bin/hdfs dfsrouteradmin -add /readonly ns1 / -readonly
+
 If a mount point is not set, the Router will map it to the default namespace `dfs.federation.router.default.nameserviceId`.
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
index 0c01763..0ead93e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/FederationTestUtils.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.federation;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
 
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
@@ -52,6 +51,9 @@ import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamenodeServi
 import org.apache.hadoop.hdfs.server.federation.resolver.NamenodeStatusReport;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import com.google.common.base.Supplier;
 
 /**
  * Helper utilities for testing HDFS Federation.
@@ -108,33 +110,41 @@ public final class FederationTestUtils {
     return report;
   }
 
-  public static void waitNamenodeRegistered(ActiveNamenodeResolver resolver,
-      String nsId, String nnId, FederationNamenodeServiceState finalState)
-          throws InterruptedException, IllegalStateException, IOException {
-
-    for (int loopCount = 0; loopCount < 20; loopCount++) {
-      if (loopCount > 0) {
-        Thread.sleep(1000);
-      }
-
-      List<? extends FederationNamenodeContext> namenodes =
-          resolver.getNamenodesForNameserviceId(nsId);
-      for (FederationNamenodeContext namenode : namenodes) {
-        // Check if this is the Namenode we are checking
-        if (namenode.getNamenodeId() == nnId  ||
-            namenode.getNamenodeId().equals(nnId)) {
-          if (finalState != null && !namenode.getState().equals(finalState)) {
-            // Wrong state, wait a bit more
-            break;
-          } else {
-            // Found and verified
-            return;
+  /**
+   * Wait for a namenode to be registered with a particular state.
+   * @param resolver Active namenode resolver.
+   * @param nsId Nameservice identifier.
+   * @param nnId Namenode identifier.
+   * @param finalState State to check for.
+   * @throws Exception Failed to verify State Store registration of namenode
+   *                   nsId:nnId for state.
+   */
+  public static void waitNamenodeRegistered(
+      final ActiveNamenodeResolver resolver,
+      final String nsId, final String nnId,
+      final FederationNamenodeServiceState state) throws Exception {
+
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        try {
+          List<? extends FederationNamenodeContext> namenodes =
+              resolver.getNamenodesForNameserviceId(nsId);
+          if (namenodes != null) {
+            for (FederationNamenodeContext namenode : namenodes) {
+              // Check if this is the Namenode we are checking
+              if (namenode.getNamenodeId() == nnId  ||
+                  namenode.getNamenodeId().equals(nnId)) {
+                return state == null || namenode.getState().equals(state);
+              }
+            }
           }
+        } catch (IOException e) {
+          // Ignore
         }
+        return false;
       }
-    }
-    fail("Failed to verify State Store registration of " + nsId + " " + nnId +
-        " for state " + finalState);
+    }, 1000, 20 * 1000);
   }
 
   public static boolean verifyDate(Date d1, Date d2, long precision) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
index 4ea0f19..7424499 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/RouterDFSCluster.java
@@ -747,8 +747,7 @@ public class RouterDFSCluster {
     }
   }
 
-  public void waitNamenodeRegistration()
-      throws InterruptedException, IllegalStateException, IOException {
+  public void waitNamenodeRegistration() throws Exception {
     for (RouterContext r : this.routers) {
       Router router = r.router;
       for (NamenodeContext nn : this.namenodes) {
@@ -761,7 +760,7 @@ public class RouterDFSCluster {
 
   public void waitRouterRegistrationQuorum(RouterContext router,
       FederationNamenodeServiceState state, String nsId, String nnId)
-          throws InterruptedException, IOException {
+          throws Exception {
     LOG.info("Waiting for NN {} {} to transition to {}", nsId, nnId, state);
     ActiveNamenodeResolver nnResolver = router.router.getNamenodeResolver();
     waitNamenodeRegistered(nnResolver, nsId, nnId, state);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
index 682d569..c92e92c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
@@ -69,6 +69,7 @@ public class TestMountTableResolver {
    * ______file1.txt -> 4:/user/file1.txt
    * __usr
    * ____bin -> 2:/bin
+   * __readonly -> 2:/tmp
    *
    * @throws IOException If it cannot set the mount table.
    */
@@ -107,6 +108,12 @@ public class TestMountTableResolver {
     // /user/a/demo/test/b
     map = getMountTableEntry("3", "/user/test");
     mountTable.addEntry(MountTable.newInstance("/user/a/demo/test/b", map));
+
+    // /readonly
+    map = getMountTableEntry("2", "/tmp");
+    MountTable readOnlyEntry = MountTable.newInstance("/readonly", map);
+    readOnlyEntry.setReadOnly(true);
+    mountTable.addEntry(readOnlyEntry);
   }
 
   @Before
@@ -152,6 +159,9 @@ public class TestMountTableResolver {
     assertEquals("3->/user/test/a",
         mountTable.getDestinationForPath("/user/test/a").toString());
 
+    assertEquals("2->/tmp/tesfile1.txt",
+        mountTable.getDestinationForPath("/readonly/tesfile1.txt").toString());
+
   }
 
   private void compareLists(List<String> list1, String[] list2) {
@@ -166,8 +176,8 @@ public class TestMountTableResolver {
 
     // Check getting all mount points (virtual and real) beneath a path
     List<String> mounts = mountTable.getMountPoints("/");
-    assertEquals(3, mounts.size());
-    compareLists(mounts, new String[] {"tmp", "user", "usr"});
+    assertEquals(4, mounts.size());
+    compareLists(mounts, new String[] {"tmp", "user", "usr", "readonly"});
 
     mounts = mountTable.getMountPoints("/user");
     assertEquals(2, mounts.size());
@@ -212,9 +222,10 @@ public class TestMountTableResolver {
 
     // Check listing the mount table records at or beneath a path
     List<MountTable> records = mountTable.getMounts("/");
-    assertEquals(8, records.size());
+    assertEquals(9, records.size());
     compareRecords(records, new String[] {"/", "/tmp", "/user", "/usr/bin",
-        "user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt"});
+        "user/a", "/user/a/demo/a", "/user/a/demo/b", "/user/b/file1.txt",
+        "readonly"});
 
     records = mountTable.getMounts("/user");
     assertEquals(5, records.size());
@@ -229,6 +240,11 @@ public class TestMountTableResolver {
     records = mountTable.getMounts("/tmp");
     assertEquals(1, records.size());
     compareRecords(records, new String[] {"/tmp"});
+
+    records = mountTable.getMounts("/readonly");
+    assertEquals(1, records.size());
+    compareRecords(records, new String[] {"/readonly"});
+    assertTrue(records.get(0).isReadOnly());
   }
 
   @Test
@@ -237,7 +253,7 @@ public class TestMountTableResolver {
 
     // 3 mount points are present /tmp, /user, /usr
     compareLists(mountTable.getMountPoints("/"),
-        new String[] {"user", "usr", "tmp"});
+        new String[] {"user", "usr", "tmp", "readonly"});
 
     // /tmp currently points to namespace 2
     assertEquals("2", mountTable.getDestinationForPath("/tmp/testfile.txt")
@@ -248,7 +264,7 @@ public class TestMountTableResolver {
 
     // Now 2 mount points are present /user, /usr
     compareLists(mountTable.getMountPoints("/"),
-        new String[] {"user", "usr"});
+        new String[] {"user", "usr", "readonly"});
 
     // /tmp no longer exists, uses default namespace for mapping /
     assertEquals("1", mountTable.getDestinationForPath("/tmp/testfile.txt")
@@ -261,7 +277,7 @@ public class TestMountTableResolver {
 
     // 3 mount points are present /tmp, /user, /usr
     compareLists(mountTable.getMountPoints("/"),
-        new String[] {"user", "usr", "tmp"});
+        new String[] {"user", "usr", "tmp", "readonly"});
 
     // /usr is virtual, uses namespace 1->/
     assertEquals("1", mountTable.getDestinationForPath("/usr/testfile.txt")
@@ -272,7 +288,7 @@ public class TestMountTableResolver {
 
     // Verify the remove failed
     compareLists(mountTable.getMountPoints("/"),
-        new String[] {"user", "usr", "tmp"});
+        new String[] {"user", "usr", "tmp", "readonly"});
   }
 
   @Test
@@ -304,7 +320,7 @@ public class TestMountTableResolver {
 
     // Initial table loaded
     testDestination();
-    assertEquals(8, mountTable.getMounts("/").size());
+    assertEquals(9, mountTable.getMounts("/").size());
 
     // Replace table with /1 and /2
     List<MountTable> records = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
index 170247f..3271d56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java
@@ -144,6 +144,37 @@ public class TestRouterAdmin {
   }
 
   @Test
+  public void testAddReadOnlyMountTable() throws IOException {
+    MountTable newEntry = MountTable.newInstance(
+        "/readonly", Collections.singletonMap("ns0", "/testdir"),
+        Time.now(), Time.now());
+    newEntry.setReadOnly(true);
+
+    RouterClient client = routerContext.getAdminClient();
+    MountTableManager mountTable = client.getMountTableManager();
+
+    // Existing mount table size
+    List<MountTable> records = getMountTableEntries(mountTable);
+    assertEquals(records.size(), mockMountTable.size());
+
+    // Add
+    AddMountTableEntryRequest addRequest =
+        AddMountTableEntryRequest.newInstance(newEntry);
+    AddMountTableEntryResponse addResponse =
+        mountTable.addMountTableEntry(addRequest);
+    assertTrue(addResponse.getStatus());
+
+    // New mount table size
+    List<MountTable> records2 = getMountTableEntries(mountTable);
+    assertEquals(records2.size(), mockMountTable.size() + 1);
+
+    // Check that we have the read only entry
+    MountTable record = getMountTableEntry("/readonly");
+    assertEquals("/readonly", record.getSourcePath());
+    assertTrue(record.isReadOnly());
+  }
+
+  @Test
   public void testRemoveMountTable() throws IOException {
 
     RouterClient client = routerContext.getAdminClient();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5cd1056a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
new file mode 100644
index 0000000..8702b3c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.NamenodeContext;
+import org.apache.hadoop.hdfs.server.federation.RouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.StateStoreDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableManager;
+import org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryRequest;
+import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntryResponse;
+import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test a router end-to-end including the MountTable.
+ */
+public class TestRouterMountTable {
+
+  private static StateStoreDFSCluster cluster;
+  private static NamenodeContext nnContext;
+  private static RouterContext routerContext;
+  private static MountTableResolver mountTable;
+
+  @BeforeClass
+  public static void globalSetUp() throws Exception {
+
+    // Build and start a federated cluster
+    cluster = new StateStoreDFSCluster(false, 1);
+    Configuration conf = new RouterConfigBuilder()
+        .stateStore()
+        .admin()
+        .rpc()
+        .build();
+    cluster.addRouterOverrides(conf);
+    cluster.startCluster();
+    cluster.startRouters();
+    cluster.waitClusterUp();
+
+    // Get the end points
+    nnContext = cluster.getRandomNamenode();
+    routerContext = cluster.getRandomRouter();
+    Router router = routerContext.getRouter();
+    mountTable = (MountTableResolver) router.getSubclusterResolver();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (cluster != null) {
+      cluster.stopRouter(routerContext);
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Test
+  public void testReadOnly() throws Exception {
+
+    // Add a read only entry
+    MountTable readOnlyEntry = MountTable.newInstance(
+        "/readonly", Collections.singletonMap("ns0", "/testdir"));
+    readOnlyEntry.setReadOnly(true);
+    assertTrue(addMountTable(readOnlyEntry));
+
+    // Add a regular entry
+    MountTable regularEntry = MountTable.newInstance(
+        "/regular", Collections.singletonMap("ns0", "/testdir"));
+    assertTrue(addMountTable(regularEntry));
+
+    // Create a folder which should show in all locations
+    final FileSystem nnFs = nnContext.getFileSystem();
+    final FileSystem routerFs = routerContext.getFileSystem();
+    assertTrue(routerFs.mkdirs(new Path("/regular/newdir")));
+
+    FileStatus dirStatusNn =
+        nnFs.getFileStatus(new Path("/testdir/newdir"));
+    assertTrue(dirStatusNn.isDirectory());
+    FileStatus dirStatusRegular =
+        routerFs.getFileStatus(new Path("/regular/newdir"));
+    assertTrue(dirStatusRegular.isDirectory());
+    FileStatus dirStatusReadOnly =
+        routerFs.getFileStatus(new Path("/readonly/newdir"));
+    assertTrue(dirStatusReadOnly.isDirectory());
+
+    // It should fail writing into a read only path
+    try {
+      routerFs.mkdirs(new Path("/readonly/newdirfail"));
+      fail("We should not be able to write into a read only mount point");
+    } catch (IOException ioe) {
+      String msg = ioe.getMessage();
+      assertTrue(msg.startsWith(
+          "/readonly/newdirfail is in a read only mount point"));
+    }
+  }
+
+  /**
+   * Add a mount table entry to the mount table through the admin API.
+   * @param entry Mount table entry to add.
+   * @return If it was succesfully added.
+   * @throws IOException Problems adding entries.
+   */
+  private boolean addMountTable(final MountTable entry) throws IOException {
+    RouterClient client = routerContext.getAdminClient();
+    MountTableManager mountTableManager = client.getMountTableManager();
+    AddMountTableEntryRequest addRequest =
+        AddMountTableEntryRequest.newInstance(entry);
+    AddMountTableEntryResponse addResponse =
+        mountTableManager.addMountTableEntry(addRequest);
+
+    // Reload the Router cache
+    mountTable.loadCache(true);
+
+    return addResponse.getStatus();
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[02/50] hadoop git commit: MAPREDUCE-6998. Moving logging APIs over to slf4j in hadoop-mapreduce-client-jobclient. Contributed by Gergely Novák.

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 06e37dd..289c17e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -24,8 +24,6 @@ import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -53,6 +51,8 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
 import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Configures and starts the MR-specific components in the YARN cluster.
@@ -64,7 +64,8 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
 
   public static final String APPJAR = JarFinder.getJar(LocalContainerLauncher.class);
 
-  private static final Log LOG = LogFactory.getLog(MiniMRYarnCluster.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MiniMRYarnCluster.class);
   private JobHistoryServer historyServer;
   private JobHistoryServerWrapper historyServerWrapper;
   private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
index 7aaaa1b..8eb673c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAMWithNonNormalizedCapabilities.java
@@ -23,8 +23,6 @@ import java.io.IOException;
 
 import org.junit.Assert;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -37,9 +35,12 @@ import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMRAMWithNonNormalizedCapabilities {
-  private static final Log LOG = LogFactory.getLog(TestMRAMWithNonNormalizedCapabilities.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRAMWithNonNormalizedCapabilities.class);
   private static FileSystem localFs;
   protected static MiniMRYarnCluster mrCluster = null;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
index a96befa..0f39168 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRAppWithCombiner.java
@@ -23,8 +23,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.CustomOutputCommitter;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -47,6 +45,8 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 @SuppressWarnings("deprecation")
 public class TestMRAppWithCombiner {
@@ -54,7 +54,8 @@ public class TestMRAppWithCombiner {
   protected static MiniMRYarnCluster mrCluster;
   private static Configuration conf = new Configuration();
   private static FileSystem localFs;
-  private static final Log LOG = LogFactory.getLog(TestMRAppWithCombiner.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRAppWithCombiner.class);
 
   static {
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 6e280ad..c8b4003 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -36,8 +36,6 @@ import java.util.jar.JarOutputStream;
 import java.util.zip.ZipEntry;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.FailingMapper;
 import org.apache.hadoop.RandomTextWriterJob;
 import org.apache.hadoop.RandomTextWriterJob.RandomInputFormat;
@@ -104,10 +102,12 @@ import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMRJobs {
 
-  private static final Log LOG = LogFactory.getLog(TestMRJobs.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestMRJobs.class);
   private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
       EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
   private static final int NUM_NODE_MGRS = 3;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
index 98a6de2..64cb253 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java
@@ -26,8 +26,6 @@ import java.util.List;
 import org.junit.Assert;
 
 import org.apache.avro.AvroRemoteException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -52,11 +50,13 @@ import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMRJobsWithHistoryService {
 
-  private static final Log LOG =
-    LogFactory.getLog(TestMRJobsWithHistoryService.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRJobsWithHistoryService.class);
 
   private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
     EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
index cbf1f39..9fa8828 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
@@ -27,8 +27,6 @@ import java.util.regex.Pattern;
 import org.junit.AfterClass;
 import org.junit.Assert;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.SleepJob;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.*;
@@ -42,11 +40,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMRJobsWithProfiler {
 
-  private static final Log LOG =
-    LogFactory.getLog(TestMRJobsWithProfiler.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMRJobsWithProfiler.class);
 
   private static final EnumSet<RMAppState> TERMINAL_RM_APP_STATES =
     EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
index 7af5922..92c65e7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMROldApiJobs.java
@@ -22,8 +22,6 @@ import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.CustomOutputCommitter;
 import org.apache.hadoop.FailMapper;
 import org.apache.hadoop.conf.Configuration;
@@ -46,10 +44,13 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestMROldApiJobs {
 
-  private static final Log LOG = LogFactory.getLog(TestMROldApiJobs.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestMROldApiJobs.class);
 
   protected static MiniMRYarnCluster mrCluster;
   private static Configuration conf = new Configuration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index 0e55529..efea709 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -26,8 +26,6 @@ import java.util.concurrent.ConcurrentMap;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -42,10 +40,13 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import static org.mockito.Mockito.*;
 
 public class TestRMNMInfo {
-  private static final Log LOG = LogFactory.getLog(TestRMNMInfo.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestRMNMInfo.class);
   private static final int NUMNODEMANAGERS = 4;
   protected static MiniMRYarnCluster mrCluster;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
index 17d82d6..fe21f07 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2;
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -48,6 +46,8 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestSpeculativeExecution {
 
@@ -82,7 +82,8 @@ public class TestSpeculativeExecution {
     }
   }
 
-  private static final Log LOG = LogFactory.getLog(TestSpeculativeExecution.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSpeculativeExecution.class);
 
   protected static MiniMRYarnCluster mrCluster;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
index e198f99..24192ea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestUberAM.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2;
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.mapreduce.Counters;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobCounter;
@@ -35,10 +33,12 @@ import org.apache.hadoop.mapreduce.TaskType;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TestUberAM extends TestMRJobs {
 
-  private static final Log LOG = LogFactory.getLog(TestUberAM.class);
+  private static final Logger LOG = LoggerFactory.getLogger(TestUberAM.class);
 
   @BeforeClass
   public static void setup() throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4cae977/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
index 8ca2e16..f339edb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testjar/UserNamePermission.java
@@ -21,28 +21,23 @@ package testjar;
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public  class UserNamePermission      
 {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(UserNamePermission.class);
 
-  private static final Log LOG = LogFactory.getLog(UserNamePermission.class);
   //This mapper will read the user name and pass in to the reducer
   public static class UserNameMapper extends Mapper<LongWritable,Text,Text,Text>
   {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[11/50] hadoop git commit: YARN-7575. NPE in scheduler UI when max-capacity is not configured. Contributed by Sunil G.

Posted by xy...@apache.org.
YARN-7575. NPE in scheduler UI when max-capacity is not configured. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db4a4a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db4a4a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db4a4a1

Branch: refs/heads/HDFS-7240
Commit: 4db4a4a165d3936d59f5e67947c6bfbc9c3270a5
Parents: daa1cdd
Author: Sunil G <su...@apache.org>
Authored: Wed Nov 29 14:50:54 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../webapp/CapacitySchedulerPage.java           |   9 +-
 .../TestAbsoluteResourceConfiguration.java      | 116 +++++++++++++++++++
 2 files changed, 121 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 3f68e07..4933d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -161,10 +161,11 @@ class CapacitySchedulerPage extends RmView {
           .__("Configured Capacity:",
               capacities.getConfiguredMinResource().toString())
           .__("Configured Max Capacity:",
-              capacities.getConfiguredMaxResource().getResource()
-                  .equals(Resources.none())
-                      ? "unlimited"
-                      : capacities.getConfiguredMaxResource().toString())
+              (capacities.getConfiguredMaxResource() == null
+                  || capacities.getConfiguredMaxResource().getResource()
+                      .equals(Resources.none()))
+                          ? "unlimited"
+                          : capacities.getConfiguredMaxResource().toString())
           .__("Effective Capacity:",
               appendPercent(capacities.getEffectiveMinResource().toString(),
                   capacities.getCapacity() / 100))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 80d16bb..5c4a12e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -511,4 +511,120 @@ public class TestAbsoluteResourceConfiguration {
 
     rm.stop();
   }
+
+  @Test
+  public void testEffectiveResourceAfterIncreasinClusterResource()
+      throws Exception {
+    // create conf with basic queue configuration.
+    CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+        false);
+    setupComplexMinMaxResourceConfig(csConf);
+
+    csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+        ResourceScheduler.class);
+
+    @SuppressWarnings("resource")
+    MockRM rm = new MockRM(csConf);
+    rm.start();
+
+    // Add few nodes
+    rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+    rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+    // Get queue object to verify min/max resource configuration.
+    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+    ParentQueue qA = (ParentQueue) cs.getQueue(QUEUEA);
+    Assert.assertNotNull(qA);
+    Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
+    Assert.assertNotNull(qB);
+    Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+    Assert.assertNotNull(qC);
+    Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+    Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+    Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+    LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+    Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+    Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+    Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+    // add new NM.
+    rm.registerNode("127.0.0.3:1234", 125 * GB, 20);
+
+    // There will be no change in effective resource when nodes are added.
+    // Since configured capacity was based on initial node capacity, a
+    // re configurations is needed to use this added capacity.
+    Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+        QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+        QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+        QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+        QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+        QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+        QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+        QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+        QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+        QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+        QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+    Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+        QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+    Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+        QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+    rm.stop();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[07/50] hadoop git commit: HADOOP-15080. Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x json-lib.

Posted by xy...@apache.org.
HADOOP-15080.  Aliyun OSS: update oss sdk from 2.8.1 to 2.8.3 to remove its dependency on Cat-x json-lib.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67b2661e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67b2661e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67b2661e

Branch: refs/heads/HDFS-7240
Commit: 67b2661e3d73a68ba7ca73b112bf6baea128631e
Parents: e411dd6
Author: Sammi Chen <sa...@intel.com>
Authored: Thu Dec 7 22:46:11 2017 +0800
Committer: Sammi Chen <sa...@intel.com>
Committed: Thu Dec 7 22:46:11 2017 +0800

----------------------------------------------------------------------
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/67b2661e/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 04b93c4..0866f3e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1165,7 +1165,7 @@
       <dependency>
         <groupId>com.aliyun.oss</groupId>
         <artifactId>aliyun-sdk-oss</artifactId>
-        <version>2.8.1</version>
+        <version>2.8.3</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.httpcomponents</groupId>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[32/50] hadoop git commit: YARN-7473. Implement Framework and policy for capacity management of auto created queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
YARN-7473. Implement Framework and policy for capacity management of auto created queues. (Suma Shivaprasad via wangda)

Change-Id: Icca7805fe12f6f7fb335effff4b121b6f7f6337b


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b38643c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b38643c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b38643c9

Branch: refs/heads/HDFS-7240
Commit: b38643c9a8dd2c53024ae830b9565a550d0ec39c
Parents: 74665e3
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:10:16 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 15:10:16 2017 -0800

----------------------------------------------------------------------
 .../monitor/SchedulingMonitor.java              |   4 +-
 .../ProportionalCapacityPreemptionPolicy.java   |   7 +-
 .../monitor/capacity/TempQueuePerPartition.java |   8 +
 .../CapacitySchedulerPlanFollower.java          |  12 +-
 .../scheduler/AbstractYarnScheduler.java        |   2 +-
 .../scheduler/YarnScheduler.java                |   2 +-
 .../capacity/AbstractAutoCreatedLeafQueue.java  | 113 +++
 .../scheduler/capacity/AbstractCSQueue.java     | 101 ++-
 .../capacity/AbstractManagedParentQueue.java    | 162 ++--
 .../capacity/AutoCreatedLeafQueue.java          | 136 ++--
 .../capacity/AutoCreatedLeafQueueConfig.java    |  66 ++
 .../AutoCreatedQueueManagementPolicy.java       |  64 ++
 .../scheduler/capacity/CSQueue.java             |   2 +-
 .../scheduler/capacity/CSQueueUtils.java        |  33 +-
 .../scheduler/capacity/CapacityScheduler.java   | 104 +--
 .../CapacitySchedulerConfiguration.java         | 112 ++-
 .../capacity/CapacitySchedulerContext.java      |   8 +
 .../capacity/CapacitySchedulerQueueManager.java |   6 +-
 .../scheduler/capacity/LeafQueue.java           |  75 +-
 .../scheduler/capacity/ManagedParentQueue.java  | 294 +++++++-
 .../scheduler/capacity/PlanQueue.java           | 150 +++-
 .../capacity/QueueManagementChange.java         | 148 ++++
 .../QueueManagementDynamicEditPolicy.java       | 272 +++++++
 .../scheduler/capacity/ReservationQueue.java    |  91 +++
 .../GuaranteedOrZeroCapacityOverTimePolicy.java | 745 +++++++++++++++++++
 .../scheduler/common/QueueEntitlement.java      |  22 +
 .../event/QueueManagementChangeEvent.java       |  49 ++
 .../scheduler/event/SchedulerEventType.java     |   5 +-
 .../capacity/TestAutoCreatedLeafQueue.java      | 113 ---
 ...stCapacitySchedulerAutoCreatedQueueBase.java | 579 ++++++++++++++
 .../TestCapacitySchedulerAutoQueueCreation.java | 611 ++++++---------
 .../TestCapacitySchedulerDynamicBehavior.java   |  32 +-
 ...tGuaranteedOrZeroCapacityOverTimePolicy.java |  40 +
 .../scheduler/capacity/TestLeafQueue.java       | 125 +++-
 .../TestQueueManagementDynamicEditPolicy.java   | 121 +++
 .../capacity/TestReservationQueue.java          | 114 +++
 36 files changed, 3629 insertions(+), 899 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
index 09edb98..d1cc850 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/SchedulingMonitor.java
@@ -79,7 +79,7 @@ public class SchedulingMonitor extends AbstractService {
   }
 
   private void schedulePreemptionChecker() {
-    handler = ses.scheduleAtFixedRate(new PreemptionChecker(),
+    handler = ses.scheduleAtFixedRate(new PolicyInvoker(),
         0, monitorInterval, TimeUnit.MILLISECONDS);
   }
 
@@ -99,7 +99,7 @@ public class SchedulingMonitor extends AbstractService {
     scheduleEditPolicy.editSchedule();
   }
 
-  private class PreemptionChecker implements Runnable {
+  private class PolicyInvoker implements Runnable {
     @Override
     public void run() {
       try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 8327cb9..304d204 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuot
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ManagedParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptableQueue;
@@ -377,7 +380,9 @@ public class ProportionalCapacityPreemptionPolicy
   }
 
   private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
-    if (q.children == null || q.children.isEmpty()) {
+    // If its a ManagedParentQueue, it might not have any children
+    if ((q.children == null || q.children.isEmpty())
+        && !(q.parentQueue instanceof ManagedParentQueue)) {
       return ImmutableSet.of(q.queueName);
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 4d71223..fdeee52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -21,6 +21,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ParentQueue;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -56,6 +59,7 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
   final ArrayList<TempQueuePerPartition> children;
   private Collection<TempAppPerPartition> apps;
   LeafQueue leafQueue;
+  ParentQueue parentQueue;
   boolean preemptionDisabled;
 
   protected Resource pendingDeductReserved;
@@ -90,6 +94,10 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
       pendingDeductReserved = Resources.createResource(0);
     }
 
+    if (ParentQueue.class.isAssignableFrom(queue.getClass())) {
+      parentQueue = (ParentQueue) queue;
+    }
+
     this.normalizedGuarantee = new double[ResourceUtils
         .getNumberOfKnownResourceTypes()];
     this.children = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
index 2e16689..7962d8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java
@@ -28,10 +28,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ReservationQueue;
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
@@ -92,8 +94,8 @@ public class CapacitySchedulerPlanFollower extends AbstractSchedulerPlanFollower
       String planQueueName, Queue queue, String currResId) {
     PlanQueue planQueue = (PlanQueue)queue;
     try {
-      AutoCreatedLeafQueue resQueue =
-          new AutoCreatedLeafQueue(cs, currResId, planQueue);
+      ReservationQueue resQueue =
+          new ReservationQueue(cs, currResId, planQueue);
       cs.addQueue(resQueue);
     } catch (SchedulerDynamicEditException e) {
       LOG.warn(
@@ -112,8 +114,8 @@ public class CapacitySchedulerPlanFollower extends AbstractSchedulerPlanFollower
     PlanQueue planQueue = (PlanQueue)queue;
     if (cs.getQueue(defReservationId) == null) {
       try {
-        AutoCreatedLeafQueue defQueue =
-            new AutoCreatedLeafQueue(cs, defReservationId, planQueue);
+        ReservationQueue defQueue =
+            new ReservationQueue(cs, defReservationId, planQueue);
         cs.addQueue(defQueue);
       } catch (SchedulerDynamicEditException e) {
         LOG.warn(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index d94efb1..cf5e13b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -457,7 +457,7 @@ public abstract class AbstractYarnScheduler
   }
 
   @Override
-  public void addQueue(Queue newQueue) throws YarnException {
+  public void addQueue(Queue newQueue) throws YarnException, IOException {
     throw new YarnException(getClass().getSimpleName()
         + " does not support this operation");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 111998b..93ca7c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -272,7 +272,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
    * @param newQueue the queue being added.
    * @throws YarnException
    */
-  void addQueue(Queue newQueue) throws YarnException;
+  void addQueue(Queue newQueue) throws YarnException, IOException;
 
   /**
    * This method increase the entitlement for current queue (must respect

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
new file mode 100644
index 0000000..ac97d72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
+    .QueueEntitlement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
+    .NO_LABEL;
+
+/**
+ * Abstract class for dynamic auto created queues managed by an implementation
+ * of AbstractManagedParentQueue
+ */
+public class AbstractAutoCreatedLeafQueue extends LeafQueue {
+
+  protected AbstractManagedParentQueue parent;
+
+  public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs,
+      String queueName, AbstractManagedParentQueue parent, CSQueue old)
+      throws IOException {
+    super(cs, queueName, parent, old);
+    this.parent = parent;
+  }
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      AbstractAutoCreatedLeafQueue.class);
+
+  public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs,
+      CapacitySchedulerConfiguration leafQueueConfigs, String queueName,
+      AbstractManagedParentQueue parent, CSQueue old) throws IOException {
+    super(cs, leafQueueConfigs, queueName, parent, old);
+    this.parent = parent;
+  }
+
+  /**
+   * This methods to change capacity for a queue and adjusts its
+   * absoluteCapacity
+   *
+   * @param entitlement the new entitlement for the queue (capacity,
+   *                    maxCapacity, etc..)
+   * @throws SchedulerDynamicEditException
+   */
+  public void setEntitlement(QueueEntitlement entitlement)
+      throws SchedulerDynamicEditException {
+     setEntitlement(NO_LABEL, entitlement);
+  }
+
+  /**
+   * This methods to change capacity for a queue and adjusts its
+   * absoluteCapacity
+   *
+   * @param entitlement the new entitlement for the queue (capacity,
+   *                    maxCapacity, etc..)
+   * @throws SchedulerDynamicEditException
+   */
+  public void setEntitlement(String nodeLabel, QueueEntitlement entitlement)
+      throws SchedulerDynamicEditException {
+    try {
+      writeLock.lock();
+      float capacity = entitlement.getCapacity();
+      if (capacity < 0 || capacity > 1.0f) {
+        throw new SchedulerDynamicEditException(
+            "Capacity demand is not in the [0,1] range: " + capacity);
+      }
+      setCapacity(nodeLabel, capacity);
+      setAbsoluteCapacity(nodeLabel,
+          getParent().getQueueCapacities().
+              getAbsoluteCapacity(nodeLabel)
+              * getQueueCapacities().getCapacity(nodeLabel));
+      // note: we currently set maxCapacity to capacity
+      // this might be revised later
+      setMaxCapacity(nodeLabel, entitlement.getMaxCapacity());
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("successfully changed to " + capacity + " for queue " + this
+            .getQueueName());
+      }
+
+      //update queue used capacity etc
+      CSQueueUtils.updateQueueStatistics(resourceCalculator,
+          csContext.getClusterResource(),
+          this, labelManager, nodeLabel);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  protected void setupConfigurableCapacities(QueueCapacities queueCapacities) {
+    CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
+        queueCapacities, parent == null ? null : parent.getQueueCapacities());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 140ea5d..4df4cf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -128,27 +128,34 @@ public abstract class AbstractCSQueue implements CSQueue {
 
   public AbstractCSQueue(CapacitySchedulerContext cs,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
+    this(cs, cs.getConfiguration(), queueName, parent, old);
+  }
+
+  public AbstractCSQueue(CapacitySchedulerContext cs,
+      CapacitySchedulerConfiguration configuration, String queueName,
+      CSQueue parent, CSQueue old) {
+
     this.labelManager = cs.getRMContext().getNodeLabelManager();
     this.parent = parent;
     this.queueName = queueName;
-    this.queuePath =
-      ((parent == null) ? "" : (parent.getQueuePath() + ".")) + this.queueName;
+    this.queuePath = ((parent == null) ? "" : (parent.getQueuePath() + "."))
+        + this.queueName;
     this.resourceCalculator = cs.getResourceCalculator();
     this.activitiesManager = cs.getActivitiesManager();
-    
+
     // must be called after parent and queueName is set
-    this.metrics =
-        old != null ? (CSQueueMetrics) old.getMetrics() : CSQueueMetrics
-            .forQueue(getQueuePath(), parent, cs.getConfiguration()
-                .getEnableUserMetrics(), cs.getConf());
+    this.metrics = old != null ?
+        (CSQueueMetrics) old.getMetrics() :
+        CSQueueMetrics.forQueue(getQueuePath(), parent,
+            configuration.getEnableUserMetrics(), cs.getConf());
 
     this.csContext = cs;
     this.minimumAllocation = csContext.getMinimumResourceCapability();
-    
+
     // initialize ResourceUsage
     queueUsage = new ResourceUsage();
     queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
-    
+
     // initialize QueueCapacities
     queueCapacities = new QueueCapacities(parent == null);
 
@@ -159,11 +166,16 @@ public abstract class AbstractCSQueue implements CSQueue {
     readLock = lock.readLock();
     writeLock = lock.writeLock();
   }
-  
+
   protected void setupConfigurableCapacities() {
+    setupConfigurableCapacities(csContext.getConfiguration());
+  }
+
+  protected void setupConfigurableCapacities(
+      CapacitySchedulerConfiguration configuration) {
     CSQueueUtils.loadUpdateAndCheckCapacities(
         getQueuePath(),
-        csContext.getConfiguration(), 
+        configuration,
         queueCapacities,
         parent == null ? null : parent.getQueueCapacities());
   }
@@ -275,6 +287,29 @@ public abstract class AbstractCSQueue implements CSQueue {
     }
   }
 
+  /**
+   * Set maximum capacity
+   * @param maximumCapacity new max capacity
+   */
+  void setMaxCapacity(String nodeLabel, float maximumCapacity) {
+    try {
+      writeLock.lock();
+      // Sanity check
+      CSQueueUtils.checkMaxCapacity(getQueueName(),
+          queueCapacities.getCapacity(nodeLabel), maximumCapacity);
+      float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
+          maximumCapacity, parent);
+      CSQueueUtils.checkAbsoluteCapacity(getQueueName(),
+          queueCapacities.getAbsoluteCapacity(nodeLabel), absMaxCapacity);
+
+      queueCapacities.setMaximumCapacity(maximumCapacity);
+      queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+
   @Override
   public String getDefaultNodeLabelExpression() {
     return defaultLabelExpression;
@@ -282,13 +317,20 @@ public abstract class AbstractCSQueue implements CSQueue {
   
   void setupQueueConfigs(Resource clusterResource)
       throws IOException {
+    setupQueueConfigs(clusterResource, csContext.getConfiguration());
+  }
+
+  protected void setupQueueConfigs(Resource clusterResource,
+      CapacitySchedulerConfiguration configuration) throws
+      IOException {
+
     try {
       writeLock.lock();
       // get labels
       this.accessibleLabels =
-          csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath());
+          configuration.getAccessibleNodeLabels(getQueuePath());
       this.defaultLabelExpression =
-          csContext.getConfiguration().getDefaultNodeLabelExpression(
+          configuration.getDefaultNodeLabelExpression(
               getQueuePath());
       this.resourceTypes = new HashSet<String>();
       for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
@@ -308,7 +350,7 @@ public abstract class AbstractCSQueue implements CSQueue {
       }
 
       // After we setup labels, we can setup capacities
-      setupConfigurableCapacities();
+      setupConfigurableCapacities(configuration);
 
       // Also fetch minimum/maximum resource constraint for this queue if
       // configured.
@@ -316,20 +358,20 @@ public abstract class AbstractCSQueue implements CSQueue {
       updateConfigurableResourceRequirement(getQueuePath(), clusterResource);
 
       this.maximumAllocation =
-          csContext.getConfiguration().getMaximumAllocationPerQueue(
+          configuration.getMaximumAllocationPerQueue(
               getQueuePath());
 
       // initialized the queue state based on previous state, configured state
       // and its parent state.
       QueueState previous = getState();
-      QueueState configuredState = csContext.getConfiguration()
+      QueueState configuredState = configuration
           .getConfiguredState(getQueuePath());
       QueueState parentState = (parent == null) ? null : parent.getState();
       initializeQueueState(previous, configuredState, parentState);
 
       authorizer = YarnAuthorizationProvider.getInstance(csContext.getConf());
 
-      this.acls = csContext.getConfiguration().getAcls(getQueuePath());
+      this.acls = configuration.getAcls(getQueuePath());
 
       // Update metrics
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
@@ -361,18 +403,21 @@ public abstract class AbstractCSQueue implements CSQueue {
       this.reservationsContinueLooking =
           csContext.getConfiguration().getReservationContinueLook();
 
-      this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
+      this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this,
+          configuration);
 
-      this.priority = csContext.getConfiguration().getQueuePriority(
+      this.priority = configuration.getQueuePriority(
           getQueuePath());
 
-      this.userWeights = getUserWeightsFromHierarchy();
+      this.userWeights = getUserWeightsFromHierarchy(configuration);
     } finally {
       writeLock.unlock();
     }
   }
 
-  private Map<String, Float> getUserWeightsFromHierarchy() throws IOException {
+  private Map<String, Float> getUserWeightsFromHierarchy
+      (CapacitySchedulerConfiguration configuration) throws
+      IOException {
     Map<String, Float> unionInheritedWeights = new HashMap<String, Float>();
     CSQueue parentQ = getParent();
     if (parentQ != null) {
@@ -381,9 +426,8 @@ public abstract class AbstractCSQueue implements CSQueue {
     }
     // Insert this queue's user's weights, overriding parent's user's weights if
     // there is overlap.
-    CapacitySchedulerConfiguration csConf = csContext.getConfiguration();
     unionInheritedWeights.putAll(
-        csConf.getAllUserWeightsForQueue(getQueuePath()));
+        configuration.getAllUserWeightsForQueue(getQueuePath()));
     return unionInheritedWeights;
   }
 
@@ -720,10 +764,11 @@ public abstract class AbstractCSQueue implements CSQueue {
    * 
    * @return true if queue has preemption disabled, false otherwise
    */
-  private boolean isQueueHierarchyPreemptionDisabled(CSQueue q) {
-    CapacitySchedulerConfiguration csConf = csContext.getConfiguration();
+  private boolean isQueueHierarchyPreemptionDisabled(CSQueue q,
+      CapacitySchedulerConfiguration configuration) {
     boolean systemWidePreemption =
-        csConf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
+        csContext.getConfiguration()
+            .getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
                        YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS);
     CSQueue parentQ = q.getParent();
 
@@ -735,14 +780,14 @@ public abstract class AbstractCSQueue implements CSQueue {
     // on, then q does not have preemption disabled (default=false, below)
     // unless the preemption_disabled property is explicitly set.
     if (parentQ == null) {
-      return csConf.getPreemptionDisabled(q.getQueuePath(), false);
+      return configuration.getPreemptionDisabled(q.getQueuePath(), false);
     }
 
     // If this is not the root queue, inherit the default value for the
     // preemption_disabled property from the parent. Preemptability will be
     // inherited from the parent's hierarchy unless explicitly overridden at
     // this level.
-    return csConf.getPreemptionDisabled(q.getQueuePath(),
+    return configuration.getPreemptionDisabled(q.getQueuePath(),
                                         parentQ.getPreemptionDisabled());
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index 46f5cf1..9d38f79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -17,13 +17,21 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
+    .QueueEntitlement;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Comparator;
 import java.util.Iterator;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 /**
  * A container class for automatically created child leaf queues.
@@ -35,13 +43,12 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
   private static final Logger LOG = LoggerFactory.getLogger(
       AbstractManagedParentQueue.class);
 
-  protected AutoCreatedLeafQueueTemplate leafQueueTemplate;
+  protected AutoCreatedLeafQueueConfig leafQueueTemplate;
+  protected AutoCreatedQueueManagementPolicy queueManagementPolicy = null;
 
   public AbstractManagedParentQueue(CapacitySchedulerContext cs,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
     super(cs, queueName, parent, old);
-
-    super.setupQueueConfigs(csContext.getClusterResource());
   }
 
   @Override
@@ -53,52 +60,18 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
       // Set new configs
       setupQueueConfigs(clusterResource);
 
-      // run reinitialize on each existing queue, to trigger absolute cap
-      // recomputations
-      for (CSQueue res : this.getChildQueues()) {
-        res.reinitialize(res, clusterResource);
-      }
     } finally {
       writeLock.unlock();
     }
   }
 
   /**
-   * Initialize leaf queue configs from template configurations specified on
-   * parent queue.
-   */
-  protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs
-    (String queuePath) {
-
-    CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-
-    AutoCreatedLeafQueueTemplate.Builder leafQueueTemplateBuilder = new
-        AutoCreatedLeafQueueTemplate.Builder();
-    int maxApps = conf.getMaximumApplicationsPerQueue(queuePath);
-    if (maxApps < 0) {
-      maxApps = (int) (
-          CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
-              * getAbsoluteCapacity());
-    }
-
-    int userLimit = conf.getUserLimit(queuePath);
-    float userLimitFactor = conf.getUserLimitFactor(queuePath);
-    leafQueueTemplateBuilder.userLimit(userLimit)
-          .userLimitFactor(userLimitFactor)
-          .maxApps(maxApps)
-          .maxAppsPerUser(
-              (int) (maxApps * (userLimit / 100.0f) * userLimitFactor));
-
-    return leafQueueTemplateBuilder;
-  }
-
-  /**
    * Add the specified child queue.
    * @param childQueue reference to the child queue to be added
    * @throws SchedulerDynamicEditException
    */
   public void addChildQueue(CSQueue childQueue)
-      throws SchedulerDynamicEditException {
+      throws SchedulerDynamicEditException, IOException {
     try {
       writeLock.lock();
       if (childQueue.getCapacity() > 0) {
@@ -193,84 +166,69 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
     }
   }
 
-  public static class AutoCreatedLeafQueueTemplate {
-
-    private QueueCapacities queueCapacities;
-
-    private int maxApps;
-    private int maxAppsPerUser;
-    private int userLimit;
-    private float userLimitFactor;
-
-    AutoCreatedLeafQueueTemplate(Builder builder) {
-      this.maxApps = builder.maxApps;
-      this.maxAppsPerUser = builder.maxAppsPerUser;
-      this.userLimit = builder.userLimit;
-      this.userLimitFactor = builder.userLimitFactor;
-      this.queueCapacities = builder.queueCapacities;
-    }
-
-    public static class Builder {
-      private int maxApps;
-      private int maxAppsPerUser;
+  public AutoCreatedLeafQueueConfig getLeafQueueTemplate() {
+    return leafQueueTemplate;
+  }
 
-      private int userLimit;
-      private float userLimitFactor;
+  public AutoCreatedQueueManagementPolicy
+  getAutoCreatedQueueManagementPolicy() {
+    return queueManagementPolicy;
+  }
 
-      private QueueCapacities queueCapacities;
+  protected SortedMap<String, String> getConfigurationsWithPrefix
+      (SortedMap<String, String> sortedConfigs, String prefix) {
+    return sortedConfigs.subMap( prefix, prefix + Character.MAX_VALUE );
+  }
 
-      Builder maxApps(int maxApplications) {
-        this.maxApps =  maxApplications;
-        return this;
-      }
+  protected SortedMap<String, String> sortCSConfigurations() {
+    SortedMap<String, String> sortedConfigs = new TreeMap(
+        new Comparator<String>() {
+          public int compare(String s1, String s2) {
+            return s1.compareToIgnoreCase(s2);
+          }
 
-      Builder maxAppsPerUser(int maxApplicationsPerUser) {
-        this.maxAppsPerUser = maxApplicationsPerUser;
-        return this;
-      }
+        });
 
-      Builder userLimit(int usrLimit) {
-        this.userLimit = usrLimit;
-        return this;
-      }
+    for (final Iterator<Map.Entry<String, String>> iterator =
+         csContext.getConfiguration().iterator(); iterator.hasNext(); ) {
+      final Map.Entry<String, String> confKeyValuePair = iterator.next();
+      sortedConfigs.put(confKeyValuePair.getKey(), confKeyValuePair.getValue());
+    }
+    return sortedConfigs;
+  }
 
-      Builder userLimitFactor(float ulf) {
-        this.userLimitFactor = ulf;
-        return this;
-      }
+  protected CapacitySchedulerConfiguration initializeLeafQueueConfigs(String
+      configPrefix) {
 
-      Builder capacities(QueueCapacities capacities) {
-        this.queueCapacities = capacities;
-        return this;
-      }
+    CapacitySchedulerConfiguration leafQueueConfigs = new
+        CapacitySchedulerConfiguration(new Configuration(false), false);
 
-      AutoCreatedLeafQueueTemplate build() {
-        return new AutoCreatedLeafQueueTemplate(this);
-      }
-    }
+    SortedMap<String, String> sortedConfigs = sortCSConfigurations();
+    SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
+        (sortedConfigs, configPrefix);
 
-    public int getUserLimit() {
-      return userLimit;
+    for (final Iterator<Map.Entry<String, String>> iterator =
+         templateConfigs.entrySet().iterator(); iterator.hasNext(); ) {
+      Map.Entry<String, String> confKeyValuePair = iterator.next();
+      leafQueueConfigs.set(confKeyValuePair.getKey(),
+          confKeyValuePair.getValue());
     }
 
-    public float getUserLimitFactor() {
-      return userLimitFactor;
-    }
+    return leafQueueConfigs;
+  }
 
-    public QueueCapacities getQueueCapacities() {
-      return queueCapacities;
-    }
+  protected void validateQueueEntitlementChange(AbstractAutoCreatedLeafQueue
+      leafQueue, QueueEntitlement entitlement)
+      throws SchedulerDynamicEditException {
 
-    public int getMaxApps() {
-      return maxApps;
-    }
+    float sumChilds = sumOfChildCapacities();
+    float newChildCap =
+        sumChilds - leafQueue.getCapacity() + entitlement.getCapacity();
 
-    public int getMaxAppsPerUser() {
-      return maxAppsPerUser;
+    if (!(newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON)) {
+      throw new SchedulerDynamicEditException(
+          "Sum of child queues should exceed 100% for auto creating parent "
+              + "queue : " + queueName);
     }
   }
-
-  public AutoCreatedLeafQueueTemplate getLeafQueueTemplate() {
-    return leafQueueTemplate;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index bc206d4..1d796ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -21,36 +21,27 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
-    .AbstractManagedParentQueue.AutoCreatedLeafQueueTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 
 /**
- * Leaf queues which are auto created by an underkying implementation of
+ * Leaf queues which are auto created by an underlying implementation of
  * AbstractManagedParentQueue. Eg: PlanQueue for reservations or
  * ManagedParentQueue for auto created dynamic queues
  */
-public class AutoCreatedLeafQueue extends LeafQueue {
+public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue {
 
   private static final Logger LOG = LoggerFactory
       .getLogger(AutoCreatedLeafQueue.class);
 
-  private AbstractManagedParentQueue parent;
-
   public AutoCreatedLeafQueue(CapacitySchedulerContext cs, String queueName,
-      AbstractManagedParentQueue parent) throws IOException {
-    super(cs, queueName, parent, null);
-
-    AutoCreatedLeafQueueTemplate leafQueueTemplate =
-        parent.getLeafQueueTemplate();
-    updateApplicationAndUserLimits(leafQueueTemplate.getUserLimit(),
-        leafQueueTemplate.getUserLimitFactor(),
-        leafQueueTemplate.getMaxApps(),
-        leafQueueTemplate.getMaxAppsPerUser());
-    this.parent = parent;
+      ManagedParentQueue parent) throws IOException {
+    super(cs, parent.getLeafQueueConfigs(queueName),
+        queueName,
+        parent, null);
+    updateCapacitiesToZero();
   }
 
   @Override
@@ -61,48 +52,75 @@ public class AutoCreatedLeafQueue extends LeafQueue {
 
       validate(newlyParsedQueue);
 
-      super.reinitialize(newlyParsedQueue, clusterResource);
-      CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
-          this, labelManager, null);
+      ManagedParentQueue managedParentQueue = (ManagedParentQueue) parent;
 
-      AutoCreatedLeafQueueTemplate leafQueueTemplate =
-          parent.getLeafQueueTemplate();
-      updateApplicationAndUserLimits(leafQueueTemplate.getUserLimit(),
-          leafQueueTemplate.getUserLimitFactor(),
-          leafQueueTemplate.getMaxApps(),
-          leafQueueTemplate.getMaxAppsPerUser());
+      super.reinitialize(newlyParsedQueue, clusterResource, managedParentQueue
+          .getLeafQueueConfigs(newlyParsedQueue.getQueueName()));
+
+      //Reset capacities to 0 since reinitialize above
+      // queueCapacities to initialize to configured capacity which might
+      // overcommit resources from parent queue
+      updateCapacitiesToZero();
 
     } finally {
       writeLock.unlock();
     }
   }
 
-  /**
-   * This methods to change capacity for a queue and adjusts its
-   * absoluteCapacity.
-   *
-   * @param entitlement the new entitlement for the queue (capacity,
-   *                    maxCapacity)
-   * @throws SchedulerDynamicEditException
-   */
-  public void setEntitlement(QueueEntitlement entitlement)
-      throws SchedulerDynamicEditException {
+  public void reinitializeFromTemplate(AutoCreatedLeafQueueConfig
+      leafQueueTemplate) throws SchedulerDynamicEditException, IOException {
+
     try {
       writeLock.lock();
-      float capacity = entitlement.getCapacity();
+
+      // TODO:
+      // reinitialize only capacities for now since 0 capacity updates
+      // can cause
+      // abs capacity related config computations to be incorrect if we go
+      // through reinitialize
+      QueueCapacities capacities = leafQueueTemplate.getQueueCapacities();
+
+      //update abs capacities
+      setupConfigurableCapacities(capacities);
+
+      //reset capacities for the leaf queue
+      mergeCapacities(capacities);
+
+      //update queue used capacity for all the node labels
+      CSQueueUtils.updateQueueStatistics(resourceCalculator,
+          csContext.getClusterResource(),
+          this, labelManager, null);
+
+      //activate applications if any are pending
+      activateApplications();
+
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void mergeCapacities(QueueCapacities capacities) {
+    for ( String nodeLabel : capacities.getExistingNodeLabels()) {
+      this.queueCapacities.setCapacity(nodeLabel,
+          capacities.getCapacity(nodeLabel));
+      this.queueCapacities.setAbsoluteCapacity(nodeLabel, capacities
+          .getAbsoluteCapacity(nodeLabel));
+      this.queueCapacities.setMaximumCapacity(nodeLabel, capacities
+          .getMaximumCapacity(nodeLabel));
+      this.queueCapacities.setAbsoluteMaximumCapacity(nodeLabel, capacities
+          .getAbsoluteMaximumCapacity(nodeLabel));
+    }
+  }
+
+  public void validateConfigurations(AutoCreatedLeafQueueConfig template)
+      throws SchedulerDynamicEditException {
+    QueueCapacities capacities = template.getQueueCapacities();
+    for (String label : capacities.getExistingNodeLabels()) {
+      float capacity = capacities.getCapacity(label);
       if (capacity < 0 || capacity > 1.0f) {
         throw new SchedulerDynamicEditException(
             "Capacity demand is not in the [0,1] range: " + capacity);
       }
-      setCapacity(capacity);
-      setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
-      setMaxCapacity(entitlement.getMaxCapacity());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("successfully changed to " + capacity + " for queue " + this
-            .getQueueName());
-      }
-    } finally {
-      writeLock.unlock();
     }
   }
 
@@ -113,22 +131,20 @@ public class AutoCreatedLeafQueue extends LeafQueue {
           "Error trying to reinitialize " + getQueuePath() + " from "
               + newlyParsedQueue.getQueuePath());
     }
-
-  }
-
-  @Override
-  protected void setupConfigurableCapacities() {
-    CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
-        queueCapacities, parent == null ? null : parent.getQueueCapacities());
   }
 
-  private void updateApplicationAndUserLimits(int userLimit,
-      float userLimitFactor,
-      int maxAppsForAutoCreatedQueues,
-      int maxAppsPerUserForAutoCreatedQueues) {
-    setUserLimit(userLimit);
-    setUserLimitFactor(userLimitFactor);
-    setMaxApplications(maxAppsForAutoCreatedQueues);
-    setMaxApplicationsPerUser(maxAppsPerUserForAutoCreatedQueues);
+  private void updateCapacitiesToZero() throws IOException {
+    try {
+      for( String nodeLabel : parent.getQueueCapacities().getExistingNodeLabels
+          ()) {
+        //TODO - update to use getMaximumCapacity(nodeLabel) in YARN-7574
+        setEntitlement(nodeLabel, new QueueEntitlement(0.0f,
+            parent.getLeafQueueTemplate()
+                .getQueueCapacities()
+                .getMaximumCapacity()));
+      }
+    } catch (SchedulerDynamicEditException e) {
+      throw new IOException(e);
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
new file mode 100644
index 0000000..5952250
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+/**
+ * Auto Created Leaf queue configurations, capacity
+ */
+public class AutoCreatedLeafQueueConfig {
+
+  /**
+   * Template queue capacities - contains configured and derived capacities
+   * like abs capacity which are used by auto queue creation policy to manage
+   * leaf queue capacities
+   */
+  private QueueCapacities queueCapacities;
+
+  private CapacitySchedulerConfiguration leafQueueConfigs;
+
+  public AutoCreatedLeafQueueConfig(Builder builder) {
+    this.queueCapacities = builder.queueCapacities;
+    this.leafQueueConfigs = builder.leafQueueConfigs;
+  }
+
+  public static class Builder {
+
+    private QueueCapacities queueCapacities;
+    private CapacitySchedulerConfiguration leafQueueConfigs;
+
+    public Builder capacities(QueueCapacities capacities) {
+      this.queueCapacities = capacities;
+      return this;
+    }
+
+    public Builder configuration(CapacitySchedulerConfiguration conf) {
+      this.leafQueueConfigs = conf;
+      return this;
+    }
+
+    public AutoCreatedLeafQueueConfig build() {
+      return new AutoCreatedLeafQueueConfig(this);
+    }
+  }
+
+  public QueueCapacities getQueueCapacities() {
+    return queueCapacities;
+  }
+
+  public CapacitySchedulerConfiguration getLeafQueueConfigs() {
+    return leafQueueConfigs;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
new file mode 100644
index 0000000..f7a4bbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueManagementPolicy.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import java.util.List;
+
+public interface AutoCreatedQueueManagementPolicy {
+
+  /**
+   * Initialize policy
+   * @param schedulerContext Capacity Scheduler context
+   */
+  void init(CapacitySchedulerContext schedulerContext, ParentQueue parentQueue);
+
+  /**
+   * Reinitialize policy state ( if required )
+   * @param schedulerContext Capacity Scheduler context
+   */
+  void reinitialize(CapacitySchedulerContext schedulerContext,
+      ParentQueue parentQueue);
+
+  /**
+   * Get initial template for the specified leaf queue
+   * @param leafQueue the leaf queue
+   * @return initial leaf queue template configurations and capacities for
+   * auto created queue
+   */
+  AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
+      AbstractAutoCreatedLeafQueue leafQueue)
+      throws SchedulerDynamicEditException;
+
+  /**
+   * Compute/Adjust child queue capacities
+   * for auto created leaf queues
+   *
+   * @return returns a list of suggested QueueEntitlementChange(s) which may
+   * or may not be be enforced by the scheduler
+   */
+  List<QueueManagementChange> computeQueueManagementChanges()
+      throws SchedulerDynamicEditException;
+
+  /**
+   * Commit/Update state for the specified queue management changes.
+   */
+  void commitQueueManagementChanges(
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 6d79b6a..5dd307c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -230,7 +230,7 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
    * @param newlyParsedQueue new queue to re-initalize from
    * @param clusterResource resources in the cluster
    */
-  public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource) 
+  public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
   throws IOException;
 
    /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 51e5b17..3901398 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -29,9 +29,9 @@ import org.apache.hadoop.yarn.util.resource.Resources;
 
 import com.google.common.collect.Sets;
 
-class CSQueueUtils {
+public class CSQueueUtils {
 
-  final static float EPSILON = 0.0001f;
+  public final static float EPSILON = 0.0001f;
   
   /*
    * Used only by tests
@@ -123,12 +123,12 @@ class CSQueueUtils {
 
     for (String label : configuredNodelabels) {
       if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
-        queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
+        queueCapacities.setCapacity(label,
             csConf.getNonLabeledQueueCapacity(queuePath) / 100);
-        queueCapacities.setMaximumCapacity(CommonNodeLabelsManager.NO_LABEL,
+        queueCapacities.setMaximumCapacity(label,
             csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100);
         queueCapacities.setMaxAMResourcePercentage(
-            CommonNodeLabelsManager.NO_LABEL,
+            label,
             csConf.getMaximumAMResourcePercentPerPartition(queuePath, label));
       } else {
         queueCapacities.setCapacity(label,
@@ -183,9 +183,32 @@ class CSQueueUtils {
 
     if (Resources.greaterThan(rc, totalPartitionResource,
         totalPartitionResource, Resources.none())) {
+
       Resource queueGuranteedResource = childQueue
           .getEffectiveCapacity(nodePartition);
 
+      //TODO : Modify below code to support Absolute Resource configurations
+      // (YARN-5881) for AutoCreatedLeafQueue
+      if (Float.compare(queueCapacities.getAbsoluteCapacity
+              (nodePartition), 0f) == 0
+          && childQueue instanceof AutoCreatedLeafQueue) {
+
+        //If absolute capacity is 0 for a leaf queue (could be a managed leaf
+        // queue, then use the leaf queue's template capacity to compute
+        // guaranteed resource for used capacity)
+
+        // queueGuaranteed = totalPartitionedResource *
+        // absolute_capacity(partition)
+        ManagedParentQueue parentQueue = (ManagedParentQueue)
+            childQueue.getParent();
+        QueueCapacities leafQueueTemplateCapacities = parentQueue
+            .getLeafQueueTemplate()
+            .getQueueCapacities();
+        queueGuranteedResource = Resources.multiply(totalPartitionResource,
+            leafQueueTemplateCapacities.getAbsoluteCapacity
+                (nodePartition));
+      }
+
       // make queueGuranteed >= minimum_allocation to avoid divided by 0.
       queueGuranteedResource =
           Resources.max(rc, totalPartitionResource, queueGuranteedResource,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index de93a6a..a5efd9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
+
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -129,6 +130,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
 
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .QueueManagementChangeEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
@@ -138,6 +141,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleC
 import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.utils.Lock;
+import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -175,6 +179,8 @@ public class CapacityScheduler extends
 
   private CSConfigurationProvider csConfProvider;
 
+  protected Clock monotonicClock;
+
   @Override
   public void setConf(Configuration conf) {
       yarnConf = conf;
@@ -1501,7 +1507,7 @@ public class CapacityScheduler extends
     {
       NodeLabelsUpdateSchedulerEvent labelUpdateEvent =
           (NodeLabelsUpdateSchedulerEvent) event;
-      
+
       updateNodeLabelsAndQueueResource(labelUpdateEvent);
     }
     break;
@@ -1613,6 +1619,25 @@ public class CapacityScheduler extends
       }
     }
     break;
+    case MANAGE_QUEUE:
+    {
+      QueueManagementChangeEvent queueManagementChangeEvent =
+          (QueueManagementChangeEvent) event;
+      ParentQueue parentQueue = queueManagementChangeEvent.getParentQueue();
+      try {
+        final List<QueueManagementChange> queueManagementChanges =
+            queueManagementChangeEvent.getQueueManagementChanges();
+        ((ManagedParentQueue) parentQueue)
+            .validateAndApplyQueueManagementChanges(queueManagementChanges);
+      } catch (SchedulerDynamicEditException sde) {
+        LOG.error("Queue Management Change event cannot be applied for "
+            + "parent queue : " + parentQueue.getQueueName(), sde);
+      } catch (IOException ioe) {
+        LOG.error("Queue Management Change event cannot be applied for "
+            + "parent queue : " + parentQueue.getQueueName(), ioe);
+      }
+    }
+    break;
     default:
       LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
     }
@@ -1976,12 +2001,14 @@ public class CapacityScheduler extends
       writeLock.lock();
       LOG.info("Removing queue: " + queueName);
       CSQueue q = this.getQueue(queueName);
-      if (!(q instanceof AutoCreatedLeafQueue)) {
+      if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(
+          q.getClass()))) {
         throw new SchedulerDynamicEditException(
             "The queue that we are asked " + "to remove (" + queueName
-                + ") is not a AutoCreatedLeafQueue");
+                + ") is not a AutoCreatedLeafQueue or ReservationQueue");
       }
-      AutoCreatedLeafQueue disposableLeafQueue = (AutoCreatedLeafQueue) q;
+      AbstractAutoCreatedLeafQueue disposableLeafQueue =
+          (AbstractAutoCreatedLeafQueue) q;
       // at this point we should have no more apps
       if (disposableLeafQueue.getNumApplications() > 0) {
         throw new SchedulerDynamicEditException(
@@ -1994,8 +2021,8 @@ public class CapacityScheduler extends
       ((AbstractManagedParentQueue) disposableLeafQueue.getParent())
           .removeChildQueue(q);
       this.queueManager.removeQueue(queueName);
-      LOG.info("Removal of AutoCreatedLeafQueue "
-          + queueName + " has succeeded");
+      LOG.info(
+          "Removal of AutoCreatedLeafQueue " + queueName + " has succeeded");
     } finally {
       writeLock.unlock();
     }
@@ -2003,22 +2030,27 @@ public class CapacityScheduler extends
 
   @Override
   public void addQueue(Queue queue)
-      throws SchedulerDynamicEditException {
+      throws SchedulerDynamicEditException, IOException {
     try {
       writeLock.lock();
-      if (!(queue instanceof AutoCreatedLeafQueue)) {
+      if (queue == null) {
+        throw new SchedulerDynamicEditException(
+            "Queue specified is null. Should be an implementation of "
+                + "AbstractAutoCreatedLeafQueue");
+      } else if (!(AbstractAutoCreatedLeafQueue.class
+          .isAssignableFrom(queue.getClass()))) {
         throw new SchedulerDynamicEditException(
-            "Queue " + queue.getQueueName() + " is not a AutoCreatedLeafQueue");
+            "Queue is not an implementation of "
+                + "AbstractAutoCreatedLeafQueue : " + queue.getClass());
       }
 
-      AutoCreatedLeafQueue newQueue = (AutoCreatedLeafQueue) queue;
+      AbstractAutoCreatedLeafQueue newQueue =
+          (AbstractAutoCreatedLeafQueue) queue;
 
-      if (newQueue.getParent() == null
-          || !(AbstractManagedParentQueue.class.
+      if (newQueue.getParent() == null || !(AbstractManagedParentQueue.class.
           isAssignableFrom(newQueue.getParent().getClass()))) {
         throw new SchedulerDynamicEditException(
-            "ParentQueue for " + newQueue.getQueueName()
-                + " is not properly set"
+            "ParentQueue for " + newQueue + " is not properly set"
                 + " (should be set and be a PlanQueue or ManagedParentQueue)");
       }
 
@@ -2027,6 +2059,7 @@ public class CapacityScheduler extends
       String queuename = newQueue.getQueueName();
       parentPlan.addChildQueue(newQueue);
       this.queueManager.addQueue(queuename, newQueue);
+
       LOG.info("Creation of AutoCreatedLeafQueue " + newQueue + " succeeded");
     } finally {
       writeLock.unlock();
@@ -2039,48 +2072,32 @@ public class CapacityScheduler extends
     try {
       writeLock.lock();
       LeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue);
-      AbstractManagedParentQueue parent = (AbstractManagedParentQueue) queue
-          .getParent();
+      AbstractManagedParentQueue parent =
+          (AbstractManagedParentQueue) queue.getParent();
 
-      if (!(queue instanceof AutoCreatedLeafQueue)) {
+      if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(
+          queue.getClass()))) {
         throw new SchedulerDynamicEditException(
             "Entitlement can not be" + " modified dynamically since queue "
                 + inQueue + " is not a AutoCreatedLeafQueue");
       }
 
-      if (parent == null
-          || !(AbstractManagedParentQueue.class.isAssignableFrom(
-              parent.getClass()))) {
+      if (parent == null || !(AbstractManagedParentQueue.class.isAssignableFrom(
+          parent.getClass()))) {
         throw new SchedulerDynamicEditException(
             "The parent of AutoCreatedLeafQueue " + inQueue
                 + " must be a PlanQueue/ManagedParentQueue");
       }
 
-      AutoCreatedLeafQueue newQueue = (AutoCreatedLeafQueue) queue;
+      AbstractAutoCreatedLeafQueue newQueue =
+          (AbstractAutoCreatedLeafQueue) queue;
+      parent.validateQueueEntitlementChange(newQueue, entitlement);
 
-      float sumChilds = parent.sumOfChildCapacities();
-      float newChildCap =
-          sumChilds - queue.getCapacity() + entitlement.getCapacity();
+      newQueue.setEntitlement(entitlement);
 
-      if (newChildCap >= 0 && newChildCap < 1.0f + CSQueueUtils.EPSILON) {
-        // note: epsilon checks here are not ok, as the epsilons might
-        // accumulate and become a problem in aggregate
-        if (Math.abs(entitlement.getCapacity() - queue.getCapacity()) == 0
-            && Math.abs(
-            entitlement.getMaxCapacity() - queue.getMaximumCapacity()) == 0) {
-          return;
-        }
-        newQueue.setEntitlement(entitlement);
-      } else{
-        throw new SchedulerDynamicEditException(
-            "Sum of child queues should exceed 100% for auto creating parent "
-                + "queue : " + parent.getQueueName());
-      }
-      LOG.info(
-          "Set entitlement for AutoCreatedLeafQueue " + inQueue
-              + "  to " + queue.getCapacity() +
-              " request was (" + entitlement.getCapacity()
-              + ")");
+      LOG.info("Set entitlement for AutoCreatedLeafQueue " + inQueue + "  to "
+          + queue.getCapacity() + " request was (" + entitlement.getCapacity()
+          + ")");
     } finally {
       writeLock.unlock();
     }
@@ -2718,7 +2735,6 @@ public class CapacityScheduler extends
 
         addQueue(autoCreatedLeafQueue);
 
-        //TODO - Set entitlement through capacity management policy
       } else{
         throw new SchedulerDynamicEditException(
             "Could not auto-create leaf queue for " + leafQueueName

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index a33d81a..8aa41ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -923,6 +923,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     return getInt(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY);
   }
 
+  @VisibleForTesting
+  public void setNodeLocalityDelay(int nodeLocalityDelay) {
+    setInt(NODE_LOCALITY_DELAY, nodeLocalityDelay);
+  }
+
   public int getRackLocalityAdditionalDelay() {
     return getInt(RACK_LOCALITY_ADDITIONAL_DELAY,
         DEFAULT_RACK_LOCALITY_ADDITIONAL_DELAY);
@@ -1401,6 +1406,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     return maxApplicationsPerQueue;
   }
 
+  public void setGlobalMaximumApplicationsPerQueue(int val) {
+    setInt(QUEUE_GLOBAL_MAX_APPLICATION, val);
+  }
+
   /**
    * Ordering policy inside a parent queue to sort queues
    */
@@ -1621,8 +1630,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public static final boolean DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED = false;
 
   @Private
+  private static final String AUTO_CREATE_CHILD_QUEUE_PREFIX =
+      "auto-create-child-queue.";
+
+  @Private
   public static final String AUTO_CREATE_CHILD_QUEUE_ENABLED =
-      "auto-create-child-queue.enabled";
+      AUTO_CREATE_CHILD_QUEUE_PREFIX + "enabled";
 
   @Private
   public static final String AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX =
@@ -1722,8 +1735,83 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   }
 
   @Private
+  public static final String AUTO_CREATED_QUEUE_MANAGEMENT_POLICY =
+      AUTO_CREATE_CHILD_QUEUE_PREFIX + "management-policy";
+
+  @Private
+  public static final String DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY =
+      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity"
+          + ".queuemanagement."
+          + "GuaranteedOrZeroCapacityOverTimePolicy";
+
+  @Private
+  private static final String QUEUE_MANAGEMENT_CONFIG_PREFIX =
+      "yarn.resourcemanager.monitor.capacity.queue-management.";
+
+  /**
+   * Time in milliseconds between invocations of this policy
+   */
+  @Private
+  public static final String QUEUE_MANAGEMENT_MONITORING_INTERVAL =
+      QUEUE_MANAGEMENT_CONFIG_PREFIX + "monitoring-interval";
+
+  @Private
+  public static final long DEFAULT_QUEUE_MANAGEMENT_MONITORING_INTERVAL =
+      1500L;
+
+  /**
+   * Queue Management computation policy for Auto Created queues
+   * @param queue The queue's path
+   * @return Configured policy class name
+   */
+  @Private
+  public String getAutoCreatedQueueManagementPolicy(String queue) {
+    String autoCreatedQueueManagementPolicy =
+        get(getQueuePrefix(queue) + AUTO_CREATED_QUEUE_MANAGEMENT_POLICY,
+            DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY);
+    return autoCreatedQueueManagementPolicy;
+  }
+
+  /**
+   * Get The policy class configured to manage capacities for auto created leaf
+   * queues under the specified parent
+   *
+   * @param queueName The parent queue's name
+   * @return The policy class configured to manage capacities for auto created
+   * leaf queues under the specified parent queue
+   */
+  @Private
+  protected AutoCreatedQueueManagementPolicy
+  getAutoCreatedQueueManagementPolicyClass(
+      String queueName) {
+
+    String queueManagementPolicyClassName =
+        getAutoCreatedQueueManagementPolicy(queueName);
+    LOG.info("Using Auto Created Queue Management Policy: "
+        + queueManagementPolicyClassName + " for queue: " + queueName);
+    try {
+      Class<?> queueManagementPolicyClazz = getClassByName(
+          queueManagementPolicyClassName);
+      if (AutoCreatedQueueManagementPolicy.class.isAssignableFrom(
+          queueManagementPolicyClazz)) {
+        return (AutoCreatedQueueManagementPolicy) ReflectionUtils.newInstance(
+            queueManagementPolicyClazz, this);
+      } else{
+        throw new YarnRuntimeException(
+            "Class: " + queueManagementPolicyClassName + " not instance of "
+                + AutoCreatedQueueManagementPolicy.class.getCanonicalName());
+      }
+    } catch (ClassNotFoundException e) {
+      throw new YarnRuntimeException(
+          "Could not instantiate " + "AutoCreatedQueueManagementPolicy: "
+              + queueManagementPolicyClassName + " for queue: " + queueName,
+          e);
+    }
+  }
+
   @VisibleForTesting
-  public void setAutoCreatedLeafQueueTemplateCapacity(String queuePath,
+  @Private
+  public void setAutoCreatedLeafQueueConfigCapacity(String queuePath,
       float val) {
     String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
         queuePath);
@@ -1732,13 +1820,31 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
   @Private
   @VisibleForTesting
-  public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
+  public void setAutoCreatedLeafQueueConfigMaxCapacity(String queuePath,
       float val) {
     String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
         queuePath);
     setMaximumCapacity(leafQueueConfPrefix, val);
   }
 
+  @VisibleForTesting
+  @Private
+  public void setAutoCreatedLeafQueueConfigUserLimit(String queuePath,
+      int val) {
+    String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+        queuePath);
+    setUserLimit(leafQueueConfPrefix, val);
+  }
+
+  @VisibleForTesting
+  @Private
+  public void setAutoCreatedLeafQueueConfigUserLimitFactor(String queuePath,
+      float val) {
+    String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+        queuePath);
+    setUserLimitFactor(leafQueueConfPrefix, val);
+  }
+
   public static String getUnits(String resourceValue) {
     String units;
     for (int i = 0; i < resourceValue.length(); i++) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 7c918a5..ae74989 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 
 /**
@@ -94,4 +95,11 @@ public interface CapacitySchedulerContext {
    * @return if configuration is mutable
    */
   boolean isConfigurationMutable();
+
+  /**
+   * Get clock from scheduler
+   * @return Clock
+   */
+  Clock getClock();
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index eb50123..30ecd40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -239,7 +239,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
             queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
 
         List<CSQueue> childQueues = new ArrayList<>();
-        AutoCreatedLeafQueue resQueue = new AutoCreatedLeafQueue(csContext,
+        ReservationQueue resQueue = new ReservationQueue(csContext,
             defReservationId, (PlanQueue) queue);
         try {
           resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
@@ -312,7 +312,8 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
       Map<String, CSQueue> newQueues) throws IOException {
     // check that all static queues are included in the newQueues list
     for (Map.Entry<String, CSQueue> e : queues.entrySet()) {
-      if (!(e.getValue() instanceof AutoCreatedLeafQueue)) {
+      if (!(AbstractAutoCreatedLeafQueue.class.isAssignableFrom(e.getValue()
+          .getClass()))) {
         String queueName = e.getKey();
         CSQueue oldQueue = e.getValue();
         CSQueue newQueue = newQueues.get(queueName);
@@ -394,7 +395,6 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
       String queueName = e.getKey();
       CSQueue existingQueue = e.getValue();
 
-      //TODO - Handle case when auto create is disabled on parent queues
       if (!newQueues.containsKey(queueName) && !(
           existingQueue instanceof AutoCreatedLeafQueue && conf
               .isAutoCreateChildQueueEnabled(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 41ec4ba..86fcbc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -138,7 +138,14 @@ public class LeafQueue extends AbstractCSQueue {
   @SuppressWarnings({ "unchecked", "rawtypes" })
   public LeafQueue(CapacitySchedulerContext cs,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
-    super(cs, queueName, parent, old);
+      this(cs, cs.getConfiguration(), queueName, parent, old);
+  }
+
+  public LeafQueue(CapacitySchedulerContext cs,
+      CapacitySchedulerConfiguration configuration,
+      String queueName, CSQueue parent, CSQueue old) throws
+      IOException {
+    super(cs, configuration, queueName, parent, old);
     this.scheduler = cs;
 
     this.usersManager = new UsersManager(metrics, this, labelManager, scheduler,
@@ -149,17 +156,25 @@ public class LeafQueue extends AbstractCSQueue {
 
     if(LOG.isDebugEnabled()) {
       LOG.debug("LeafQueue:" + " name=" + queueName
-        + ", fullname=" + getQueuePath());
+          + ", fullname=" + getQueuePath());
     }
 
-    setupQueueConfigs(cs.getClusterResource());
+    setupQueueConfigs(cs.getClusterResource(), configuration);
+
   }
 
   protected void setupQueueConfigs(Resource clusterResource)
       throws IOException {
+    setupQueueConfigs(clusterResource, csContext.getConfiguration());
+  }
+
+  protected void setupQueueConfigs(Resource clusterResource,
+      CapacitySchedulerConfiguration conf) throws
+      IOException {
     try {
       writeLock.lock();
-      super.setupQueueConfigs(clusterResource);
+      CapacitySchedulerConfiguration schedConf = csContext.getConfiguration();
+      super.setupQueueConfigs(clusterResource, conf);
 
       this.lastClusterResource = clusterResource;
 
@@ -173,8 +188,6 @@ public class LeafQueue extends AbstractCSQueue {
       // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
       setQueueResourceLimitsInfo(clusterResource);
 
-      CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-
       setOrderingPolicy(
           conf.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePath()));
 
@@ -183,11 +196,13 @@ public class LeafQueue extends AbstractCSQueue {
 
       maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
       if (maxApplications < 0) {
-        int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
+        int maxGlobalPerQueueApps = schedConf
+            .getGlobalMaximumApplicationsPerQueue();
         if (maxGlobalPerQueueApps > 0) {
           maxApplications = maxGlobalPerQueueApps;
         } else {
-          int maxSystemApps = conf.getMaximumSystemApplications();
+          int maxSystemApps = schedConf.
+              getMaximumSystemApplications();
           maxApplications =
               (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
         }
@@ -218,9 +233,11 @@ public class LeafQueue extends AbstractCSQueue {
                         .join(getAccessibleNodeLabels().iterator(), ',')));
       }
 
-      nodeLocalityDelay = conf.getNodeLocalityDelay();
-      rackLocalityAdditionalDelay = conf.getRackLocalityAdditionalDelay();
-      rackLocalityFullReset = conf.getRackLocalityFullReset();
+      nodeLocalityDelay = schedConf.getNodeLocalityDelay();
+      rackLocalityAdditionalDelay = schedConf
+          .getRackLocalityAdditionalDelay();
+      rackLocalityFullReset = schedConf
+          .getRackLocalityFullReset();
 
       // re-init this since max allocation could have changed
       this.minimumAllocationFactor = Resources.ratio(resourceCalculator,
@@ -507,10 +524,11 @@ public class LeafQueue extends AbstractCSQueue {
     }
   }
 
-  @Override
-  public void reinitialize(
-      CSQueue newlyParsedQueue, Resource clusterResource) 
-  throws IOException {
+  protected void reinitialize(
+      CSQueue newlyParsedQueue, Resource clusterResource,
+      CapacitySchedulerConfiguration configuration) throws
+      IOException {
+
     try {
       writeLock.lock();
       // Sanity check
@@ -535,7 +553,7 @@ public class LeafQueue extends AbstractCSQueue {
             + newMax);
       }
 
-      setupQueueConfigs(clusterResource);
+      setupQueueConfigs(clusterResource, configuration);
 
       // queue metrics are updated, more resource may be available
       // activate the pending applications if possible
@@ -547,6 +565,14 @@ public class LeafQueue extends AbstractCSQueue {
   }
 
   @Override
+  public void reinitialize(
+      CSQueue newlyParsedQueue, Resource clusterResource)
+  throws IOException {
+    reinitialize(newlyParsedQueue, clusterResource,
+        csContext.getConfiguration());
+  }
+
+  @Override
   public void submitApplicationAttempt(FiCaSchedulerApp application,
       String userName) {
     // Careful! Locking order is important!
@@ -731,7 +757,7 @@ public class LeafQueue extends AbstractCSQueue {
     }
   }
 
-  private void activateApplications() {
+  protected void activateApplications() {
     try {
       writeLock.lock();
       // limit of allowed resource usage for application masters
@@ -1991,10 +2017,18 @@ public class LeafQueue extends AbstractCSQueue {
     queueCapacities.setCapacity(capacity);
   }
 
+  public void setCapacity(String nodeLabel, float capacity) {
+    queueCapacities.setCapacity(nodeLabel, capacity);
+  }
+
   public void setAbsoluteCapacity(float absoluteCapacity) {
     queueCapacities.setAbsoluteCapacity(absoluteCapacity);
   }
 
+  public void setAbsoluteCapacity(String nodeLabel, float absoluteCapacity) {
+    queueCapacities.setAbsoluteCapacity(nodeLabel, absoluteCapacity);
+  }
+
   public void setMaxApplicationsPerUser(int maxApplicationsPerUser) {
     this.maxApplicationsPerUser = maxApplicationsPerUser;
   }
@@ -2002,7 +2036,12 @@ public class LeafQueue extends AbstractCSQueue {
   public void setMaxApplications(int maxApplications) {
     this.maxApplications = maxApplications;
   }
-  
+
+  public void setMaxAMResourcePerQueuePercent(
+      float maxAMResourcePerQueuePercent) {
+    this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
+  }
+
   public OrderingPolicy<FiCaSchedulerApp>
       getOrderingPolicy() {
     return orderingPolicy;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[48/50] hadoop git commit: YARN-7565. Yarn service pre-maturely releases the container after AM restart. Contributed by Chandni Singh

Posted by xy...@apache.org.
YARN-7565. Yarn service pre-maturely releases the container after AM restart. Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ebe6a78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ebe6a78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ebe6a78

Branch: refs/heads/HDFS-7240
Commit: 3ebe6a7819292ce6bd557e36137531b59890c845
Parents: 06f0eb2
Author: Jian He <ji...@apache.org>
Authored: Tue Dec 12 13:35:56 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Dec 12 13:35:56 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/service/ServiceScheduler.java   |  72 +++++++++-
 .../yarn/service/component/Component.java       |   8 ++
 .../component/instance/ComponentInstance.java   |  10 +-
 .../yarn/service/conf/YarnServiceConf.java      |  10 ++
 .../hadoop/yarn/service/MockServiceAM.java      | 144 ++++++++++++++++---
 .../hadoop/yarn/service/TestServiceAM.java      |  98 ++++++++++++-
 .../yarn/client/api/async/AMRMClientAsync.java  |  10 ++
 .../api/async/impl/AMRMClientAsyncImpl.java     |   5 +
 .../types/yarn/YarnRegistryAttributes.java      |   1 +
 .../markdown/yarn-service/Configurations.md     |   1 +
 10 files changed, 330 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index bea31cf..2697050 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventT
 import org.apache.hadoop.yarn.service.component.Component;
 import org.apache.hadoop.yarn.service.component.ComponentEvent;
 import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
 import org.apache.hadoop.yarn.service.provider.ProviderUtils;
@@ -137,6 +138,9 @@ public class ServiceScheduler extends CompositeService {
   private YarnRegistryViewForProviders yarnRegistryOperations;
   private ServiceContext context;
   private ContainerLaunchService containerLaunchService;
+  private final Map<ContainerId, ComponentInstance> unRecoveredInstances =
+      new ConcurrentHashMap<>();
+  private long containerRecoveryTimeout;
 
   public ServiceScheduler(ServiceContext context) {
     super(context.service.getName());
@@ -212,6 +216,9 @@ public class ServiceScheduler extends CompositeService {
     createConfigFileCache(context.fs.getFileSystem());
 
     createAllComponents();
+    containerRecoveryTimeout = getConfig().getInt(
+        YarnServiceConf.CONTAINER_RECOVERY_TIMEOUT_MS,
+        YarnServiceConf.DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS);
   }
 
   protected YarnRegistryViewForProviders createYarnRegistryOperations(
@@ -320,7 +327,7 @@ public class ServiceScheduler extends CompositeService {
     }
     for (Container container : containersFromPrevAttempt) {
       LOG.info("Handling {} from previous attempt", container.getId());
-      ServiceRecord record = existingRecords.get(RegistryPathUtils
+      ServiceRecord record = existingRecords.remove(RegistryPathUtils
           .encodeYarnID(container.getId().toString()));
       if (record != null) {
         Component comp = componentsById.get(container.getAllocationRequestId());
@@ -337,6 +344,40 @@ public class ServiceScheduler extends CompositeService {
         amRMClient.releaseAssignedContainer(container.getId());
       }
     }
+
+    existingRecords.forEach((encodedContainerId, record) -> {
+      String componentName = record.get(YarnRegistryAttributes.YARN_COMPONENT);
+      if (componentName != null) {
+        Component component = componentsByName.get(componentName);
+        ComponentInstance compInstance = component.getComponentInstance(
+            record.description);
+        ContainerId containerId = ContainerId.fromString(record.get(
+            YarnRegistryAttributes.YARN_ID));
+        unRecoveredInstances.put(containerId, compInstance);
+        component.removePendingInstance(compInstance);
+      }
+    });
+
+    if (unRecoveredInstances.size() > 0) {
+      executorService.schedule(() -> {
+        synchronized (unRecoveredInstances) {
+          // after containerRecoveryTimeout, all the containers that haven't be
+          // recovered by the RM will released. The corresponding Component
+          // Instances are added to the pending queues of their respective
+          // component.
+          unRecoveredInstances.forEach((containerId, instance) -> {
+            LOG.info("{}, wait on container {} expired",
+                instance.getCompInstanceId(), containerId);
+            instance.cleanupRegistryAndCompHdfsDir(containerId);
+            Component component = componentsByName.get(instance.getCompName());
+            component.requestContainers(1);
+            component.reInsertPendingInstance(instance);
+            amRMClient.releaseAssignedContainer(containerId);
+          });
+          unRecoveredInstances.clear();
+        }
+      }, containerRecoveryTimeout, TimeUnit.MILLISECONDS);
+    }
   }
 
   private void initGlobalTokensForSubstitute(ServiceContext context) {
@@ -521,6 +562,35 @@ public class ServiceScheduler extends CompositeService {
       }
     }
 
+
+    @Override
+    public void onContainersReceivedFromPreviousAttempts(
+        List<Container> containers) {
+      if (containers == null || containers.isEmpty()) {
+        return;
+      }
+      for (Container container : containers) {
+        ComponentInstance compInstance;
+        synchronized (unRecoveredInstances) {
+          compInstance = unRecoveredInstances.remove(container.getId());
+        }
+        if (compInstance != null) {
+          Component component = componentsById.get(
+              container.getAllocationRequestId());
+          ComponentEvent event = new ComponentEvent(component.getName(),
+              CONTAINER_RECOVERED)
+              .setInstance(compInstance)
+              .setContainerId(container.getId())
+              .setContainer(container);
+          component.handle(event);
+        } else {
+          LOG.info("Not waiting to recover container {}, releasing",
+              container.getId());
+          amRMClient.releaseAssignedContainer(container.getId());
+        }
+      }
+    }
+
     @Override
     public void onContainersCompleted(List<ContainerStatus> statuses) {
       for (ContainerStatus status : statuses) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 4e05e5f..5189ab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -107,6 +107,10 @@ public class Component implements EventHandler<ComponentEvent> {
           .addTransition(INIT, INIT, CONTAINER_RECOVERED,
               new ContainerRecoveredTransition())
 
+          // container recovered in AM heartbeat
+          .addTransition(FLEXING, FLEXING, CONTAINER_RECOVERED,
+              new ContainerRecoveredTransition())
+
           // container allocated by RM
           .addTransition(FLEXING, FLEXING, CONTAINER_ALLOCATED,
               new ContainerAllocatedTransition())
@@ -309,6 +313,10 @@ public class Component implements EventHandler<ComponentEvent> {
     }
   }
 
+  public void removePendingInstance(ComponentInstance instance) {
+    pendingInstances.remove(instance);
+  }
+
   public void reInsertPendingInstance(ComponentInstance instance) {
     pendingInstances.add(instance);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 509f667..31fa5c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -39,15 +39,15 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.service.ServiceScheduler;
 import org.apache.hadoop.yarn.service.api.records.ContainerState;
 import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
 import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.ServiceUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -62,7 +62,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import static org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes.*;
 import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
 import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
 import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState.*;
 
@@ -398,6 +397,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
     record.set(YARN_PERSISTENCE, PersistencePolicies.CONTAINER);
     record.set(YARN_IP, status.getIPs().get(0));
     record.set(YARN_HOSTNAME, status.getHost());
+    record.set(YARN_COMPONENT, component.getName());
     try {
       yarnRegistry
           .putComponent(RegistryPathUtils.encodeYarnID(containerId), record);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index ea8904a..22926e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.service.api.records.Configuration;
 
 public class YarnServiceConf {
 
+  private static final String YARN_SERVICE_PREFIX = "yarn.service.";
+
   // Retry settings for the ServiceClient to talk to Service AppMaster
   public static final String CLIENT_AM_RETRY_MAX_WAIT_MS = "yarn.service.client-am.retry.max-wait-ms";
   public static final String CLIENT_AM_RETRY_MAX_INTERVAL_MS = "yarn.service.client-am.retry-interval-ms";
@@ -84,6 +86,14 @@ public class YarnServiceConf {
   public static final String JVM_OPTS = "yarn.service.am.java.opts";
 
   /**
+   * How long to wait until a container is considered dead.
+   */
+  public static final String CONTAINER_RECOVERY_TIMEOUT_MS =
+      YARN_SERVICE_PREFIX + "container-recovery.timeout.ms";
+
+  public static final int DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS = 120000;
+
+  /**
    * Get long value for the property. First get from the userConf, if not
    * present, get from systemConf.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
index 4298161..37b18fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -19,8 +19,13 @@
 package org.apache.hadoop.yarn.service;
 
 import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
@@ -42,15 +47,24 @@ import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
 import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeoutException;
 
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class MockServiceAM extends ServiceMaster {
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(MockServiceAM.class);
+
   Service service;
   // The list of containers fed by tests to be returned on
   // AMRMClientCallBackHandler#onContainersAllocated
@@ -59,6 +73,16 @@ public class MockServiceAM extends ServiceMaster {
 
   final List<ContainerStatus> failedContainers =
       Collections.synchronizedList(new LinkedList<>());
+
+  private final List<Container> recoveredContainers =
+      Collections.synchronizedList(new LinkedList<>());
+
+  private final Map<String, ServiceRecord> registryComponents =
+      new ConcurrentHashMap<>();
+
+  private Map<ContainerId, ContainerStatus> containerStatuses =
+      new ConcurrentHashMap<>();
+
   public MockServiceAM(Service service) {
     super(service.getName());
     this.service = service;
@@ -75,7 +99,7 @@ public class MockServiceAM extends ServiceMaster {
   @Override
   protected Path getAppDir() {
     Path path = new Path(new Path("target", "apps"), service.getName());
-    System.out.println("Service path: " + path);
+    LOG.info("Service path: {}", path);
     return path;
   }
 
@@ -84,10 +108,24 @@ public class MockServiceAM extends ServiceMaster {
       throws IOException, YarnException {
     return new ServiceScheduler(context) {
 
+      @SuppressWarnings("SuspiciousMethodCalls")
       @Override
       protected YarnRegistryViewForProviders createYarnRegistryOperations(
           ServiceContext context, RegistryOperations registryClient) {
-        return mock(YarnRegistryViewForProviders.class);
+        YarnRegistryViewForProviders yarnRegistryView = mock(
+            YarnRegistryViewForProviders.class);
+        if (!registryComponents.isEmpty()) {
+          try {
+            when(yarnRegistryView.listComponents())
+                .thenReturn(new LinkedList<>(registryComponents.keySet()));
+            when(yarnRegistryView.getComponent(anyString())).thenAnswer(
+                invocation ->
+                    registryComponents.get(invocation.getArguments()[0]));
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        }
+        return yarnRegistryView;
       }
 
       @Override
@@ -101,7 +139,7 @@ public class MockServiceAM extends ServiceMaster {
             // add new containers if any
             synchronized (feedContainers) {
               if (feedContainers.isEmpty()) {
-                System.out.println("Allocating........ no containers");
+                LOG.info("Allocating........ no containers");
               } else {
                 // The AMRMClient will return containers for compoenent that are
                 // at FLEXING state
@@ -112,7 +150,7 @@ public class MockServiceAM extends ServiceMaster {
                   org.apache.hadoop.yarn.service.component.Component component =
                       componentsById.get(c.getAllocationRequestId());
                   if (component.getState() == ComponentState.FLEXING) {
-                    System.out.println("Allocated container " + c.getId());
+                    LOG.info("Allocated container {} ", c.getId());
                     allocatedContainers.add(c);
                     itor.remove();
                   }
@@ -121,6 +159,17 @@ public class MockServiceAM extends ServiceMaster {
               }
             }
 
+            // add recovered containers if any
+            synchronized (recoveredContainers) {
+              if (!recoveredContainers.isEmpty()) {
+                List<Container> containersFromPrevAttempt = new LinkedList<>();
+                containersFromPrevAttempt.addAll(recoveredContainers);
+                recoveredContainers.clear();
+                builder.containersFromPreviousAttempt(
+                    containersFromPrevAttempt);
+              }
+            }
+
             // add failed containers if any
             synchronized (failedContainers) {
               if (!failedContainers.isEmpty()) {
@@ -146,15 +195,23 @@ public class MockServiceAM extends ServiceMaster {
           }
         };
 
-        return AMRMClientAsync
-            .createAMRMClientAsync(client1, 1000,
+        return AMRMClientAsync.createAMRMClientAsync(client1, 1000,
                 this.new AMRMClientCallback());
       }
 
+      @SuppressWarnings("SuspiciousMethodCalls")
       @Override
       public NMClientAsync createNMClient() {
         NMClientAsync nmClientAsync = super.createNMClient();
-        nmClientAsync.setClient(mock(NMClient.class));
+        NMClient nmClient = mock(NMClient.class);
+        try {
+          when(nmClient.getContainerStatus(anyObject(), anyObject()))
+              .thenAnswer(invocation ->
+                  containerStatuses.get(invocation.getArguments()[0]));
+        } catch (YarnException | IOException e) {
+          throw new RuntimeException(e);
+        }
+        nmClientAsync.setClient(nmClient);
         return nmClientAsync;
       }
     };
@@ -165,6 +222,33 @@ public class MockServiceAM extends ServiceMaster {
     context.service = service;
   }
 
+  public void feedRegistryComponent(ContainerId containerId, String compName,
+      String compInstName) {
+    ServiceRecord record = new ServiceRecord();
+    record.set(YarnRegistryAttributes.YARN_ID, containerId.toString());
+    record.description = compInstName;
+    record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+        PersistencePolicies.CONTAINER);
+    record.set(YarnRegistryAttributes.YARN_IP, "localhost");
+    record.set(YarnRegistryAttributes.YARN_HOSTNAME, "localhost");
+    record.set(YarnRegistryAttributes.YARN_COMPONENT, compName);
+    registryComponents.put(RegistryPathUtils.encodeYarnID(
+        containerId.toString()), record);
+  }
+
+  /**
+   * Simulates a recovered container that is sent to the AM in the heartbeat
+   * response.
+   *
+   * @param containerId The ID for the container
+   * @param compName    The component to which the recovered container is fed.
+   */
+  public void feedRecoveredContainer(ContainerId containerId, String compName) {
+    Container container = createContainer(containerId, compName);
+    recoveredContainers.add(container);
+    addContainerStatus(container, ContainerState.RUNNING);
+  }
+
   /**
    *
    * @param service The service for the component
@@ -174,20 +258,12 @@ public class MockServiceAM extends ServiceMaster {
    */
   public Container feedContainerToComp(Service service, int id,
       String compName) {
-    ApplicationId applicationId = ApplicationId.fromString(service.getId());
-    ContainerId containerId = ContainerId
-        .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
-    NodeId nodeId = NodeId.newInstance("localhost", 1234);
-    Container container = Container
-        .newInstance(containerId, nodeId, "localhost",
-            Resource.newInstance(100, 1), Priority.newInstance(0), null);
-
-    long allocateId =
-        context.scheduler.getAllComponents().get(compName).getAllocateId();
-    container.setAllocationRequestId(allocateId);
+    ContainerId containerId = createContainerId(id);
+    Container container = createContainer(containerId, compName);
     synchronized (feedContainers) {
       feedContainers.add(container);
     }
+    addContainerStatus(container, ContainerState.RUNNING);
     return container;
   }
 
@@ -196,13 +272,30 @@ public class MockServiceAM extends ServiceMaster {
     ApplicationId applicationId = ApplicationId.fromString(service.getId());
     ContainerId containerId = ContainerId
         .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
-    ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
-    containerStatus.setContainerId(containerId);
+    ContainerStatus status = Records.newRecord(ContainerStatus.class);
+    status.setContainerId(containerId);
     synchronized (failedContainers) {
-      failedContainers.add(containerStatus);
+      failedContainers.add(status);
     }
   }
 
+  public ContainerId createContainerId(int id) {
+    ApplicationId applicationId = ApplicationId.fromString(service.getId());
+    return ContainerId.newContainerId(
+        ApplicationAttemptId.newInstance(applicationId, 1), id);
+  }
+
+  private Container createContainer(ContainerId containerId, String compName) {
+    NodeId nodeId = NodeId.newInstance("localhost", 1234);
+    Container container = Container.newInstance(
+        containerId, nodeId, "localhost",
+        Resource.newInstance(100, 1),
+        Priority.newInstance(0), null);
+    long allocateId =
+        context.scheduler.getAllComponents().get(compName).getAllocateId();
+    container.setAllocationRequestId(allocateId);
+    return container;
+  }
 
   public void flexComponent(String compName, long numberOfContainers)
       throws IOException {
@@ -256,4 +349,13 @@ public class MockServiceAM extends ServiceMaster {
       }
     }, 1000, 20000);
   }
+
+  private void addContainerStatus(Container container, ContainerState state) {
+    ContainerStatus status = ContainerStatus.newInstance(container.getId(),
+        state, "", 0);
+    status.setHost(container.getNodeId().getHost());
+    status.setIPs(Lists.newArrayList(container.getNodeId().getHost()));
+    containerStatuses.put(container.getId(), status);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
index fb4de0d..2a3303e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
@@ -20,15 +20,21 @@ package org.apache.hadoop.yarn.service;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.ComponentState;
 import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
 import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -39,6 +45,9 @@ import static org.apache.hadoop.registry.client.api.RegistryConstants
 
 public class TestServiceAM extends ServiceTestUtils{
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestServiceAM.class);
+
   private File basedir;
   YarnConfiguration conf = new YarnConfiguration();
   TestingCluster zkCluster;
@@ -54,7 +63,7 @@ public class TestServiceAM extends ServiceTestUtils{
     zkCluster = new TestingCluster(1);
     zkCluster.start();
     conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
-    System.out.println("ZK cluster: " +  zkCluster.getConnectString());
+    LOG.info("ZK cluster: {}", zkCluster.getConnectString());
   }
 
   @After
@@ -91,7 +100,7 @@ public class TestServiceAM extends ServiceTestUtils{
     am.feedContainerToComp(exampleApp, 1, "compa");
     am.waitForCompInstanceState(compa0, ComponentInstanceState.STARTED);
 
-    System.out.println("Fail the container 1");
+    LOG.info("Fail the container 1");
     // fail the container
     am.feedFailedContainerToComp(exampleApp, 1, "compa");
 
@@ -106,4 +115,89 @@ public class TestServiceAM extends ServiceTestUtils{
         am.getComponent("compa").getPendingInstances().size());
     am.stop();
   }
+
+  // Test to verify that the containers of previous attempt are not prematurely
+  // released. These containers are sent by the RM to the AM in the
+  // heartbeat response.
+  @Test(timeout = 200000)
+  public void testContainersFromPreviousAttemptsWithRMRestart()
+      throws Exception {
+    ApplicationId applicationId = ApplicationId.newInstance(
+        System.currentTimeMillis(), 1);
+    Service exampleApp = new Service();
+    exampleApp.setId(applicationId.toString());
+    exampleApp.setName("testContainersRecovers");
+    String comp1Name = "comp1";
+    String comp1InstName = "comp1-0";
+
+    org.apache.hadoop.yarn.service.api.records.Component compA =
+        createComponent(comp1Name, 1, "sleep");
+    exampleApp.addComponent(compA);
+
+    MockServiceAM am = new MockServiceAM(exampleApp);
+    ContainerId containerId = am.createContainerId(1);
+    am.feedRegistryComponent(containerId, comp1Name, comp1InstName);
+    am.init(conf);
+    am.start();
+
+    ComponentInstance comp10 = am.getCompInstance(comp1Name, comp1InstName);
+    am.feedRecoveredContainer(containerId, comp1Name);
+    am.waitForCompInstanceState(comp10, ComponentInstanceState.STARTED);
+
+    // 0 pending instance
+    Assert.assertEquals(0,
+        am.getComponent(comp1Name).getPendingInstances().size());
+
+    GenericTestUtils.waitFor(() -> am.getCompInstance(comp1Name, comp1InstName)
+        .getContainerStatus() != null, 2000, 200000);
+
+    Assert.assertEquals("container state",
+        org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+        am.getCompInstance(comp1Name, comp1InstName).getContainerStatus()
+            .getState());
+    am.stop();
+  }
+
+  // Test to verify that the containers of previous attempt are released and the
+  // component instance is added to the pending queue when the recovery wait
+  // time interval elapses.
+  @Test(timeout = 200000)
+  public void testContainersReleasedWhenExpired()
+      throws Exception {
+    ApplicationId applicationId = ApplicationId.newInstance(
+        System.currentTimeMillis(), 1);
+    Service exampleApp = new Service();
+    exampleApp.setId(applicationId.toString());
+    exampleApp.setName("testContainersRecovers");
+    String comp1Name = "comp1";
+    String comp1InstName = "comp1-0";
+
+    org.apache.hadoop.yarn.service.api.records.Component compA =
+        createComponent(comp1Name, 1, "sleep");
+    exampleApp.addComponent(compA);
+
+    MockServiceAM am = new MockServiceAM(exampleApp);
+    ContainerId containerId = am.createContainerId(1);
+    am.feedRegistryComponent(containerId, comp1Name, comp1InstName);
+    conf.setLong(YarnServiceConf.CONTAINER_RECOVERY_TIMEOUT_MS, 10);
+    am.init(conf);
+    am.start();
+    Thread.sleep(100);
+    GenericTestUtils.waitFor(() -> am.getComponent(comp1Name).getState().equals(
+        ComponentState.FLEXING), 100, 2000);
+
+    // 1 pending instance
+    Assert.assertEquals(1,
+        am.getComponent(comp1Name).getPendingInstances().size());
+
+    am.feedContainerToComp(exampleApp, 2, comp1Name);
+
+    GenericTestUtils.waitFor(() -> am.getCompInstance(comp1Name, comp1InstName)
+        .getContainerStatus() != null, 2000, 200000);
+    Assert.assertEquals("container state",
+        org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+        am.getCompInstance(comp1Name, comp1InstName).getContainerStatus()
+            .getState());
+    am.stop();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index 793ad79..a3d7959 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -486,6 +486,14 @@ extends AbstractService {
      * stop() is the recommended action.
      */
     public abstract void onError(Throwable e);
+
+    /**
+     * Called when the ResourceManager responds to a heartbeat with containers
+     * from previous attempt.
+     */
+    public void onContainersReceivedFromPreviousAttempts(
+        List<Container> containers) {
+    }
   }
 
   /**
@@ -531,5 +539,7 @@ extends AbstractService {
      * @param e
      */
     void onError(Throwable e);
+
+    void onContainersReceivedFromPreviousAttempts(List<Container> containers);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 031cdec..cafb153 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -358,6 +358,11 @@ extends AMRMClientAsync<T> {
           if (!allocated.isEmpty()) {
             handler.onContainersAllocated(allocated);
           }
+
+          if (!response.getContainersFromPreviousAttempts().isEmpty()) {
+            handler.onContainersReceivedFromPreviousAttempts(
+                response.getContainersFromPreviousAttempts());
+          }
           progress = handler.getProgress();
         } catch (Throwable ex) {
           handler.onError(ex);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
index 5eaa9c0..b6e7a20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
@@ -38,4 +38,5 @@ public final class YarnRegistryAttributes {
   public static final String YARN_PATH = "yarn:path";
   public static final String YARN_HOSTNAME = "yarn:hostname";
   public static final String YARN_IP = "yarn:ip";
+  public static final String YARN_COMPONENT = "yarn:component";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index 561df7f..a6fd998 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -125,6 +125,7 @@ Above config make the service AM to be retried at max 10 times.
 |yarn.service.log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs after the application completes. If the log file name matches both include and exclude pattern, this file will be excluded.
 |yarn.service.rolling-log.include-pattern| The regex expression for including log files whose file name matches it when aggregating the logs while app is running.
 |yarn.service.rolling-log.exclude-pattern| The regex expression for excluding log files whose file name matches it when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded.
+|yarn.service.container-recovery.timeout.ms| The timeout in milliseconds after which the service AM releases all the containers of previous attempt which are not yet recovered by the RM. By default, it is set to 120000, i.e. 2 minutes.
 
 ## Constant variables for custom service
 The service framework provides some constant variables for user to configure their services. These variables are either dynamically generated by the system or are static ones such as service name defined by the user.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[31/50] hadoop git commit: YARN-7473. Implement Framework and policy for capacity management of auto created queues. (Suma Shivaprasad via wangda)

Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
index ff795e4..cbdb21d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -17,13 +17,23 @@
  */
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler
     .SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
+    .FiCaSchedulerApp;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Auto Creation enabled Parent queue. This queue initially does not have any
@@ -44,54 +54,125 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
       final String queueName, final CSQueue parent, final CSQueue old)
       throws IOException {
     super(cs, queueName, parent, old);
-    String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
-        csContext.getConfiguration());
-    this.leafQueueTemplate = initializeLeafQueueConfigs(
-        leafQueueTemplateConfPrefix).build();
+
+    shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
+        csContext.getConfiguration()
+            .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
+                getQueuePath());
+
+    leafQueueTemplate = initializeLeafQueueConfigs().build();
 
     StringBuffer queueInfo = new StringBuffer();
     queueInfo.append("Created Managed Parent Queue: ").append(queueName).append(
         "]\nwith capacity: [").append(super.getCapacity()).append(
         "]\nwith max capacity: [").append(super.getMaximumCapacity()).append(
-        "\nwith max apps: [").append(leafQueueTemplate.getMaxApps()).append(
-        "]\nwith max apps per user: [").append(
-        leafQueueTemplate.getMaxAppsPerUser()).append("]\nwith user limit: [")
-        .append(leafQueueTemplate.getUserLimit()).append(
-        "]\nwith user limit factor: [").append(
-        leafQueueTemplate.getUserLimitFactor()).append("].");
+        "].");
     LOG.info(queueInfo.toString());
+
+    initializeQueueManagementPolicy();
   }
 
   @Override
   public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
       throws IOException {
-    validate(newlyParsedQueue);
-    super.reinitialize(newlyParsedQueue, clusterResource);
-    String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
-        csContext.getConfiguration());
-    this.leafQueueTemplate = initializeLeafQueueConfigs(
-        leafQueueTemplateConfPrefix).build();
+
+    try {
+      writeLock.lock();
+      validate(newlyParsedQueue);
+
+      shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
+          csContext.getConfiguration()
+              .getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
+                  getQueuePath());
+
+      //validate if capacity is exceeded for child queues
+      if (shouldFailAutoCreationWhenGuaranteedCapacityExceeded) {
+        float childCap = sumOfChildCapacities();
+        if (getCapacity() < childCap) {
+          throw new IOException(
+              "Total of Auto Created leaf queues guaranteed capacity : "
+                  + childCap + " exceeds Parent queue's " + getQueuePath()
+                  + " guaranteed capacity " + getCapacity() + ""
+                  + ".Cannot enforce policy to auto"
+                  + " create queues beyond parent queue's capacity");
+        }
+      }
+
+      leafQueueTemplate = initializeLeafQueueConfigs().build();
+
+      super.reinitialize(newlyParsedQueue, clusterResource);
+
+      // run reinitialize on each existing queue, to trigger absolute cap
+      // recomputations
+      for (CSQueue res : this.getChildQueues()) {
+        res.reinitialize(res, clusterResource);
+      }
+
+      //clear state in policy
+      reinitializeQueueManagementPolicy();
+
+      //reassign capacities according to policy
+      final List<QueueManagementChange> queueManagementChanges =
+          queueManagementPolicy.computeQueueManagementChanges();
+
+      validateAndApplyQueueManagementChanges(queueManagementChanges);
+
+      StringBuffer queueInfo = new StringBuffer();
+      queueInfo.append("Reinitialized Managed Parent Queue: ").append(queueName)
+          .append("]\nwith capacity: [").append(super.getCapacity()).append(
+          "]\nwith max capacity: [").append(super.getMaximumCapacity()).append(
+          "].");
+      LOG.info(queueInfo.toString());
+    } catch (YarnException ye) {
+      LOG.error("Exception while computing policy changes for leaf queue : "
+          + getQueueName(), ye);
+      throw new IOException(ye);
+    } finally {
+      writeLock.unlock();
+    }
   }
 
-  @Override
-  protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs(
-      String queuePath) {
+  private void initializeQueueManagementPolicy() {
+    queueManagementPolicy =
+        csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
+            getQueuePath());
+
+    queueManagementPolicy.init(csContext, this);
+  }
+
+  private void reinitializeQueueManagementPolicy() {
+    AutoCreatedQueueManagementPolicy managementPolicy =
+        csContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
+            getQueuePath());
+
+    if (!(managementPolicy.getClass().equals(
+        this.queueManagementPolicy.getClass()))) {
+      queueManagementPolicy = managementPolicy;
+      queueManagementPolicy.init(csContext, this);
+    } else{
+      queueManagementPolicy.reinitialize(csContext, this);
+    }
+  }
+
+  protected AutoCreatedLeafQueueConfig.Builder initializeLeafQueueConfigs() {
 
-    AutoCreatedLeafQueueTemplate.Builder leafQueueTemplate =
-        super.initializeLeafQueueConfigs(queuePath);
+    AutoCreatedLeafQueueConfig.Builder builder =
+        new AutoCreatedLeafQueueConfig.Builder();
 
-    CapacitySchedulerConfiguration conf = csContext.getConfiguration();
-    String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(conf);
+    String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
+        csContext.getConfiguration());
+    //Load template configuration
+    builder.configuration(
+        super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix));
+
+    //Load template capacities
     QueueCapacities queueCapacities = new QueueCapacities(false);
-    CSQueueUtils.loadUpdateAndCheckCapacities(leafQueueTemplateConfPrefix,
+    CSQueueUtils.loadUpdateAndCheckCapacities(csContext.getConfiguration()
+            .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
         csContext.getConfiguration(), queueCapacities, getQueueCapacities());
-    leafQueueTemplate.capacities(queueCapacities);
-
-    shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
-        conf.getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
-            getQueuePath());
+    builder.capacities(queueCapacities);
 
-    return leafQueueTemplate;
+    return builder;
   }
 
   protected void validate(final CSQueue newlyParsedQueue) throws IOException {
@@ -106,7 +187,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
 
   @Override
   public void addChildQueue(CSQueue childQueue)
-      throws SchedulerDynamicEditException {
+      throws SchedulerDynamicEditException, IOException {
     try {
       writeLock.lock();
 
@@ -138,21 +219,164 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
           throw new SchedulerDynamicEditException(
               "Cannot auto create leaf queue " + leafQueueName + ". Child "
                   + "queues capacities have reached parent queue : "
-                  + parentQueue.getQueuePath() + " guaranteed capacity");
+                  + parentQueue.getQueuePath() + "'s guaranteed capacity");
         }
       }
 
       AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue;
       super.addChildQueue(leafQueue);
-      //TODO - refresh policy queue after capacity management is added
+      final AutoCreatedLeafQueueConfig initialLeafQueueTemplate =
+          queueManagementPolicy.getInitialLeafQueueConfiguration(leafQueue);
 
+      leafQueue.reinitializeFromTemplate(initialLeafQueueTemplate);
     } finally {
       writeLock.unlock();
     }
   }
 
-  private String getLeafQueueConfigPrefix(CapacitySchedulerConfiguration conf) {
-    return conf.getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
+  public List<FiCaSchedulerApp> getScheduleableApplications() {
+    try {
+      readLock.lock();
+      List<FiCaSchedulerApp> apps = new ArrayList<>();
+      for (CSQueue childQueue : getChildQueues()) {
+        apps.addAll(((LeafQueue) childQueue).getApplications());
+      }
+      return Collections.unmodifiableList(apps);
+    } finally {
+      readLock.unlock();
+    }
   }
 
+  public List<FiCaSchedulerApp> getPendingApplications() {
+    try {
+      readLock.lock();
+      List<FiCaSchedulerApp> apps = new ArrayList<>();
+      for (CSQueue childQueue : getChildQueues()) {
+        apps.addAll(((LeafQueue) childQueue).getPendingApplications());
+      }
+      return Collections.unmodifiableList(apps);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public List<FiCaSchedulerApp> getAllApplications() {
+    try {
+      readLock.lock();
+      List<FiCaSchedulerApp> apps = new ArrayList<>();
+      for (CSQueue childQueue : getChildQueues()) {
+        apps.addAll(((LeafQueue) childQueue).getAllApplications());
+      }
+      return Collections.unmodifiableList(apps);
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  public String getLeafQueueConfigPrefix(CapacitySchedulerConfiguration conf) {
+    return CapacitySchedulerConfiguration.PREFIX + conf
+        .getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
+  }
+
+  public boolean shouldFailAutoCreationWhenGuaranteedCapacityExceeded() {
+    return shouldFailAutoCreationWhenGuaranteedCapacityExceeded;
+  }
+
+  /**
+   * Asynchronously called from scheduler to apply queue management changes
+   *
+   * @param queueManagementChanges
+   */
+  public void validateAndApplyQueueManagementChanges(
+      List<QueueManagementChange> queueManagementChanges)
+      throws IOException, SchedulerDynamicEditException {
+    try {
+      writeLock.lock();
+
+      validateQueueManagementChanges(queueManagementChanges);
+
+      applyQueueManagementChanges(queueManagementChanges);
+
+      AutoCreatedQueueManagementPolicy policy =
+          getAutoCreatedQueueManagementPolicy();
+
+      //acquires write lock on policy
+      policy.commitQueueManagementChanges(queueManagementChanges);
+
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  public void validateQueueManagementChanges(
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException {
+
+    for (QueueManagementChange queueManagementChange : queueManagementChanges) {
+
+      CSQueue childQueue = queueManagementChange.getQueue();
+
+      if (!(childQueue instanceof AutoCreatedLeafQueue)) {
+        throw new SchedulerDynamicEditException(
+            "queue should be " + "AutoCreatedLeafQueue. Found " + childQueue
+                .getClass());
+      }
+
+      if (!(AbstractManagedParentQueue.class.
+          isAssignableFrom(childQueue.getParent().getClass()))) {
+        LOG.error("Queue " + getQueueName()
+            + " is not an instance of PlanQueue or ManagedParentQueue." + " "
+            + "Ignoring update " + queueManagementChanges);
+        throw new SchedulerDynamicEditException(
+            "Queue " + getQueueName() + " is not a AutoEnabledParentQueue."
+                + " Ignoring update " + queueManagementChanges);
+      }
+
+      switch (queueManagementChange.getQueueAction()){
+      case UPDATE_QUEUE:
+        AutoCreatedLeafQueueConfig template =
+            queueManagementChange.getUpdatedQueueTemplate();
+        ((AutoCreatedLeafQueue) childQueue).validateConfigurations(template);
+        break;
+      }
+    }
+  }
+
+  private void applyQueueManagementChanges(
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException, IOException {
+    for (QueueManagementChange queueManagementChange : queueManagementChanges) {
+      switch (queueManagementChange.getQueueAction()){
+      case UPDATE_QUEUE:
+        AutoCreatedLeafQueue childQueueToBeUpdated =
+            (AutoCreatedLeafQueue) queueManagementChange.getQueue();
+        //acquires write lock on leaf queue
+        childQueueToBeUpdated.reinitializeFromTemplate(
+            queueManagementChange.getUpdatedQueueTemplate());
+        break;
+      }
+    }
+  }
+
+  public CapacitySchedulerConfiguration getLeafQueueConfigs(
+      String leafQueueName) {
+    return getLeafQueueConfigs(getLeafQueueTemplate().getLeafQueueConfigs(),
+        leafQueueName);
+  }
+
+  public CapacitySchedulerConfiguration getLeafQueueConfigs(
+      CapacitySchedulerConfiguration templateConfig, String leafQueueName) {
+    CapacitySchedulerConfiguration leafQueueConfigTemplate = new
+        CapacitySchedulerConfiguration(new Configuration(false), false);
+    for (final Iterator<Map.Entry<String, String>> iterator =
+         templateConfig.iterator(); iterator.hasNext(); ) {
+      Map.Entry<String, String> confKeyValuePair = iterator.next();
+      final String name = confKeyValuePair.getKey().replaceFirst(
+          CapacitySchedulerConfiguration
+              .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX,
+          leafQueueName);
+      leafQueueConfigTemplate.set(name, confKeyValuePair.getValue());
+    }
+    return leafQueueConfigTemplate;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index b7f8aa6..757002f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,61 +36,132 @@ public class PlanQueue extends AbstractManagedParentQueue {
 
   private static final Logger LOG = LoggerFactory.getLogger(PlanQueue.class);
 
+  private int maxAppsForReservation;
+  private int maxAppsPerUserForReservation;
+  private int userLimit;
+  private float userLimitFactor;
+  protected CapacitySchedulerContext schedulerContext;
   private boolean showReservationsAsQueues;
 
   public PlanQueue(CapacitySchedulerContext cs, String queueName,
       CSQueue parent, CSQueue old) throws IOException {
     super(cs, queueName, parent, old);
-    this.leafQueueTemplate = initializeLeafQueueConfigs(getQueuePath()).build();
+
+    this.schedulerContext = cs;
+    // Set the reservation queue attributes for the Plan
+    CapacitySchedulerConfiguration conf = cs.getConfiguration();
+    String queuePath = super.getQueuePath();
+    int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath);
+    showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath);
+    if (maxAppsForReservation < 0) {
+      maxAppsForReservation =
+          (int) (CapacitySchedulerConfiguration.
+              DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS * super
+              .getAbsoluteCapacity());
+    }
+    int userLimit = conf.getUserLimit(queuePath);
+    float userLimitFactor = conf.getUserLimitFactor(queuePath);
+    int maxAppsPerUserForReservation =
+        (int) (maxAppsForReservation * (userLimit / 100.0f) * userLimitFactor);
+    updateQuotas(userLimit, userLimitFactor, maxAppsForReservation,
+        maxAppsPerUserForReservation);
 
     StringBuffer queueInfo = new StringBuffer();
-    queueInfo.append("Created Plan Queue: ").append(queueName).append(
-        "]\nwith capacity: [").append(super.getCapacity()).append(
-        "]\nwith max capacity: [").append(super.getMaximumCapacity()).append(
-        "\nwith max apps: [").append(leafQueueTemplate.getMaxApps()).append(
-        "]\nwith max apps per user: [").append(
-        leafQueueTemplate.getMaxAppsPerUser()).append("]\nwith user limit: [")
-        .append(leafQueueTemplate.getUserLimit()).append(
-        "]\nwith user limit factor: [").append(
-        leafQueueTemplate.getUserLimitFactor()).append("].");
+    queueInfo.append("Created Plan Queue: ").append(queueName)
+        .append("\nwith capacity: [").append(super.getCapacity())
+        .append("]\nwith max capacity: [").append(super.getMaximumCapacity())
+        .append("\nwith max reservation apps: [").append(maxAppsForReservation)
+        .append("]\nwith max reservation apps per user: [")
+        .append(maxAppsPerUserForReservation).append("]\nwith user limit: [")
+        .append(userLimit).append("]\nwith user limit factor: [")
+        .append(userLimitFactor).append("].");
     LOG.info(queueInfo.toString());
   }
 
   @Override
-  public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
-      throws IOException {
-    validate(newlyParsedQueue);
-    super.reinitialize(newlyParsedQueue, clusterResource);
-    this.leafQueueTemplate = initializeLeafQueueConfigs(getQueuePath()).build();
+  public void reinitialize(CSQueue newlyParsedQueue,
+      Resource clusterResource) throws IOException {
+    try {
+      writeLock.lock();
+      // Sanity check
+      if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
+          .getQueuePath().equals(getQueuePath())) {
+        throw new IOException(
+            "Trying to reinitialize " + getQueuePath() + " from "
+                + newlyParsedQueue.getQueuePath());
+      }
+
+      PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
+
+      if (newlyParsedParentQueue.getChildQueues().size() != 1) {
+        throw new IOException(
+            "Reservable Queue should not have sub-queues in the"
+                + "configuration expect the default reservation queue");
+      }
+
+      // Set new configs
+      setupQueueConfigs(clusterResource);
+
+      updateQuotas(newlyParsedParentQueue.userLimit,
+          newlyParsedParentQueue.userLimitFactor,
+          newlyParsedParentQueue.maxAppsForReservation,
+          newlyParsedParentQueue.maxAppsPerUserForReservation);
+
+      // run reinitialize on each existing queue, to trigger absolute cap
+      // recomputations
+      for (CSQueue res : this.getChildQueues()) {
+        res.reinitialize(res, clusterResource);
+      }
+      showReservationsAsQueues =
+          newlyParsedParentQueue.showReservationsAsQueues;
+    } finally {
+      writeLock.unlock();
+    }
   }
 
-  @Override
-  protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs
-      (String queuePath) {
-    AutoCreatedLeafQueueTemplate.Builder leafQueueTemplate = super
-        .initializeLeafQueueConfigs
-        (queuePath);
-    showReservationsAsQueues = csContext.getConfiguration()
-        .getShowReservationAsQueues(queuePath);
-    return leafQueueTemplate;
+  private void updateQuotas(int userLimit, float userLimitFactor,
+      int maxAppsForReservation, int maxAppsPerUserForReservation) {
+    this.userLimit = userLimit;
+    this.userLimitFactor = userLimitFactor;
+    this.maxAppsForReservation = maxAppsForReservation;
+    this.maxAppsPerUserForReservation = maxAppsPerUserForReservation;
   }
 
-  protected void validate(final CSQueue newlyParsedQueue) throws IOException {
-    // Sanity check
-    if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
-        .getQueuePath().equals(getQueuePath())) {
-      throw new IOException(
-          "Trying to reinitialize " + getQueuePath() + " from "
-              + newlyParsedQueue.getQueuePath());
-    }
+  /**
+   * Number of maximum applications for each of the reservations in this Plan.
+   *
+   * @return maxAppsForreservation
+   */
+  public int getMaxApplicationsForReservations() {
+    return maxAppsForReservation;
+  }
 
-    PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
+  /**
+   * Number of maximum applications per user for each of the reservations in
+   * this Plan.
+   *
+   * @return maxAppsPerUserForreservation
+   */
+  public int getMaxApplicationsPerUserForReservation() {
+    return maxAppsPerUserForReservation;
+  }
 
-    if (newlyParsedParentQueue.getChildQueues().size() != 1) {
-      throw new IOException(
-          "Reservable Queue should not have sub-queues in the"
-              + "configuration expect the default reservation queue");
-    }
+  /**
+   * User limit value for each of the reservations in this Plan.
+   *
+   * @return userLimit
+   */
+  public int getUserLimitForReservation() {
+    return userLimit;
+  }
+
+  /**
+   * User limit factor value for each of the reservations in this Plan.
+   *
+   * @return userLimitFactor
+   */
+  public float getUserLimitFactor() {
+    return userLimitFactor;
   }
 
   /**
@@ -98,4 +170,4 @@ public class PlanQueue extends AbstractManagedParentQueue {
   public boolean showReservationsAsQueues() {
     return showReservationsAsQueues;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
new file mode 100644
index 0000000..74d9b23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.QueueState;
+
+/**
+ * Encapsulates Queue entitlement and state updates needed
+ * for adjusting capacity dynamically
+ *
+ */
+@Private
+@Unstable
+public abstract class QueueManagementChange {
+
+  private final CSQueue queue;
+
+  /**
+   * Updating the queue may involve entitlement updates
+   * and/or QueueState changes
+   *
+   * QueueAction can potentially be enhanced
+   * for adding, removing queues for queue management
+   */
+  public enum QueueAction {
+    UPDATE_QUEUE
+  }
+
+  private AutoCreatedLeafQueueConfig
+      queueTemplateUpdate;
+
+  private final QueueAction queueAction;
+  /**
+   * Updated Queue state with the new entitlement
+   */
+  private QueueState transitionToQueueState;
+
+  public QueueManagementChange(final CSQueue queue,
+      final QueueAction queueAction) {
+    this.queue = queue;
+    this.queueAction = queueAction;
+  }
+
+  public QueueManagementChange(final CSQueue queue,
+      final QueueAction queueAction, QueueState targetQueueState,
+      final AutoCreatedLeafQueueConfig
+          queueTemplateUpdates) {
+    this(queue, queueAction, queueTemplateUpdates);
+    this.transitionToQueueState = targetQueueState;
+  }
+
+  public QueueManagementChange(final CSQueue queue,
+      final QueueAction queueAction,
+      final AutoCreatedLeafQueueConfig
+      queueTemplateUpdates) {
+    this(queue, queueAction);
+    this.queueTemplateUpdate = queueTemplateUpdates;
+  }
+
+  public QueueState getTransitionToQueueState() {
+    return transitionToQueueState;
+  }
+
+  public CSQueue getQueue() {
+    return queue;
+  }
+
+  public AutoCreatedLeafQueueConfig getUpdatedQueueTemplate() {
+    return queueTemplateUpdate;
+  }
+
+  public QueueAction getQueueAction() {
+    return queueAction;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if (!(o instanceof QueueManagementChange))
+      return false;
+
+    QueueManagementChange that = (QueueManagementChange) o;
+
+    if (queue != null ? !queue.equals(that.queue) : that.queue != null)
+      return false;
+    if (queueTemplateUpdate != null ? !queueTemplateUpdate.equals(
+        that.queueTemplateUpdate) : that.queueTemplateUpdate != null)
+      return false;
+    if (queueAction != that.queueAction)
+      return false;
+    return transitionToQueueState == that.transitionToQueueState;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = queue != null ? queue.hashCode() : 0;
+    result = 31 * result + (queueTemplateUpdate != null ?
+        queueTemplateUpdate.hashCode() :
+        0);
+    result = 31 * result + (queueAction != null ? queueAction.hashCode() : 0);
+    result = 31 * result + (transitionToQueueState != null ?
+        transitionToQueueState.hashCode() :
+        0);
+    return result;
+  }
+
+  @Override
+  public String toString() {
+    return "QueueManagementChange{" + "queue=" + queue
+        + ", updatedEntitlementsByPartition=" + queueTemplateUpdate
+        + ", queueAction=" + queueAction + ", transitionToQueueState="
+        + transitionToQueueState + '}';
+  }
+
+  public static class UpdateQueue extends QueueManagementChange {
+
+    public UpdateQueue(final CSQueue queue, QueueState targetQueueState,
+        final AutoCreatedLeafQueueConfig
+            queueTemplateUpdate) {
+      super(queue, QueueAction.UPDATE_QUEUE, targetQueueState,
+          queueTemplateUpdate);
+    }
+
+    public UpdateQueue(final CSQueue queue,
+        final AutoCreatedLeafQueueConfig
+            queueTemplateUpdate) {
+      super(queue, QueueAction.UPDATE_QUEUE, queueTemplateUpdate);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
new file mode 100644
index 0000000..9b0cf7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event
+    .QueueManagementChangeEvent;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Queue Management scheduling policy for managed parent queues which enable
+ * auto child queue creation
+ */
+public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
+
+  private static final Log LOG =
+      LogFactory.getLog(QueueManagementDynamicEditPolicy.class);
+
+  private Clock clock;
+
+  // Pointer to other RM components
+  private RMContext rmContext;
+  private ResourceCalculator rc;
+  private CapacityScheduler scheduler;
+  private RMNodeLabelsManager nlm;
+
+  private long monitoringInterval;
+
+  private Set<String> managedParentQueues = new HashSet<>();
+
+  /**
+   * Instantiated by CapacitySchedulerConfiguration
+   */
+  public QueueManagementDynamicEditPolicy() {
+    clock = SystemClock.getInstance();
+  }
+
+  @SuppressWarnings("unchecked")
+  @VisibleForTesting
+  public QueueManagementDynamicEditPolicy(RMContext context,
+      CapacityScheduler scheduler) {
+    init(context.getYarnConfiguration(), context, scheduler);
+  }
+
+  @SuppressWarnings("unchecked")
+  @VisibleForTesting
+  public QueueManagementDynamicEditPolicy(RMContext context,
+      CapacityScheduler scheduler, Clock clock) {
+    init(context.getYarnConfiguration(), context, scheduler);
+    this.clock = clock;
+  }
+
+  @Override
+  public void init(final Configuration config, final RMContext context,
+      final ResourceScheduler sched) {
+    LOG.info("Queue Management Policy monitor:" + this.
+        getClass().getCanonicalName());
+    assert null == scheduler : "Unexpected duplicate call to init";
+    if (!(sched instanceof CapacityScheduler)) {
+      throw new YarnRuntimeException("Class " +
+          sched.getClass().getCanonicalName() + " not instance of " +
+          CapacityScheduler.class.getCanonicalName());
+    }
+    rmContext = context;
+    scheduler = (CapacityScheduler) sched;
+    clock = scheduler.getClock();
+
+    rc = scheduler.getResourceCalculator();
+    nlm = scheduler.getRMContext().getNodeLabelManager();
+
+    CapacitySchedulerConfiguration csConfig = scheduler.getConfiguration();
+
+    monitoringInterval = csConfig.getLong(
+        CapacitySchedulerConfiguration.QUEUE_MANAGEMENT_MONITORING_INTERVAL,
+        CapacitySchedulerConfiguration.
+            DEFAULT_QUEUE_MANAGEMENT_MONITORING_INTERVAL);
+
+    initQueues();
+  }
+
+  /**
+   * Reinitializes queues(Called on scheduler.reinitialize)
+   * @param config Configuration
+   * @param context The resourceManager's context
+   * @param sched The scheduler
+   */
+  public void reinitialize(final Configuration config, final RMContext context,
+      final ResourceScheduler sched) {
+    //TODO - Wire with scheduler reinitialize and remove initQueues below?
+    initQueues();
+  }
+
+  private void initQueues() {
+    managedParentQueues.clear();
+    for (Map.Entry<String, CSQueue> queues : scheduler
+        .getCapacitySchedulerQueueManager()
+        .getQueues().entrySet()) {
+
+      String queueName = queues.getKey();
+      CSQueue queue = queues.getValue();
+
+      if ( queue instanceof ManagedParentQueue) {
+        managedParentQueues.add(queueName);
+      }
+    }
+  }
+
+  @Override
+  public void editSchedule() {
+    long startTs = clock.getTime();
+
+    initQueues();
+    manageAutoCreatedLeafQueues();
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Total time used=" + (clock.getTime() - startTs) + " ms.");
+    }
+  }
+
+  @VisibleForTesting
+  List<QueueManagementChange> manageAutoCreatedLeafQueues()
+  {
+
+    List<QueueManagementChange> queueManagementChanges = new ArrayList<>();
+    // All partitions to look at
+
+    //Proceed only if there are queues to process
+    if (managedParentQueues.size() > 0) {
+      for (String parentQueueName : managedParentQueues) {
+        ManagedParentQueue parentQueue =
+            (ManagedParentQueue) scheduler.getCapacitySchedulerQueueManager().
+                getQueue(parentQueueName);
+
+        queueManagementChanges.addAll(
+            computeQueueManagementChanges
+            (parentQueue));
+      }
+    }
+    return queueManagementChanges;
+  }
+
+
+  @VisibleForTesting
+  List<QueueManagementChange> computeQueueManagementChanges
+      (ManagedParentQueue parentQueue) {
+
+    List<QueueManagementChange> queueManagementChanges =
+        Collections.emptyList();
+    if (!parentQueue.shouldFailAutoCreationWhenGuaranteedCapacityExceeded()) {
+
+      AutoCreatedQueueManagementPolicy policyClazz =
+          parentQueue.getAutoCreatedQueueManagementPolicy();
+      long startTime = 0;
+      try {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(MessageFormat
+              .format("Trying to use {0} to compute preemption "
+                      + "candidates",
+                  policyClazz.getClass().getName()));
+          startTime = clock.getTime();
+        }
+
+        queueManagementChanges = policyClazz.computeQueueManagementChanges();
+
+        //Scheduler update is asynchronous
+        if (queueManagementChanges.size() > 0) {
+          QueueManagementChangeEvent queueManagementChangeEvent =
+              new QueueManagementChangeEvent(parentQueue,
+                  queueManagementChanges);
+          scheduler.getRMContext().getDispatcher().getEventHandler().handle(
+              queueManagementChangeEvent);
+        }
+
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(MessageFormat.format("{0} uses {1} millisecond"
+                  + " to run",
+              policyClazz.getClass().getName(), clock.getTime()
+                  - startTime));
+          if (queueManagementChanges.size() > 0) {
+            LOG.debug(" Updated queue management updates for parent queue"
+                + " ["
+                + parentQueue.getQueueName() + ": [\n" + queueManagementChanges
+                .toString() + "\n]");
+          }
+        }
+      } catch (YarnException e) {
+        LOG.error(
+            "Could not compute child queue management updates for parent "
+                + "queue "
+                + parentQueue.getQueueName(), e);
+      }
+    } else{
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(
+            "Skipping queue management updates for parent queue "
+                + parentQueue
+                .getQueuePath() + " "
+                + "since configuration for  auto creating queue's beyond "
+                + "parent's "
+                + "guaranteed capacity is disabled");
+      }
+    }
+    return queueManagementChanges;
+  }
+
+  @Override
+  public long getMonitoringInterval() {
+    return monitoringInterval;
+  }
+
+  @Override
+  public String getPolicyName() {
+    return "QueueManagementDynamicEditPolicy";
+  }
+
+  public ResourceCalculator getResourceCalculator() {
+    return rc;
+  }
+
+  public RMContext getRmContext() {
+    return rmContext;
+  }
+
+  public ResourceCalculator getRC() {
+    return rc;
+  }
+
+  public CapacityScheduler getScheduler() {
+    return scheduler;
+  }
+
+  public Set<String> getManagedParentQueues() {
+    return managedParentQueues;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
new file mode 100644
index 0000000..34f4aa1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This represents a dynamic {@link LeafQueue} managed by the
+ * {@link ReservationSystem}
+ *
+ */
+public class ReservationQueue extends AbstractAutoCreatedLeafQueue {
+
+  private static final Logger LOG = LoggerFactory
+      .getLogger(ReservationQueue.class);
+
+  private PlanQueue parent;
+
+  public ReservationQueue(CapacitySchedulerContext cs, String queueName,
+      PlanQueue parent) throws IOException {
+    super(cs, queueName, parent, null);
+    // the following parameters are common to all reservation in the plan
+    updateQuotas(parent.getUserLimitForReservation(),
+        parent.getUserLimitFactor(),
+        parent.getMaxApplicationsForReservations(),
+        parent.getMaxApplicationsPerUserForReservation());
+    this.parent = parent;
+  }
+
+  @Override
+  public void reinitialize(CSQueue newlyParsedQueue,
+      Resource clusterResource) throws IOException {
+    try {
+      writeLock.lock();
+      // Sanity check
+      if (!(newlyParsedQueue instanceof ReservationQueue) || !newlyParsedQueue
+          .getQueuePath().equals(getQueuePath())) {
+        throw new IOException(
+            "Trying to reinitialize " + getQueuePath() + " from "
+                + newlyParsedQueue.getQueuePath());
+      }
+      super.reinitialize(newlyParsedQueue, clusterResource);
+      CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+          this, labelManager, null);
+
+      updateQuotas(parent.getUserLimitForReservation(),
+          parent.getUserLimitFactor(),
+          parent.getMaxApplicationsForReservations(),
+          parent.getMaxApplicationsPerUserForReservation());
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void updateQuotas(int userLimit, float userLimitFactor,
+      int maxAppsForReservation, int maxAppsPerUserForReservation) {
+    setUserLimit(userLimit);
+    setUserLimitFactor(userLimitFactor);
+    setMaxApplications(maxAppsForReservation);
+    maxApplicationsPerUser = maxAppsPerUserForReservation;
+  }
+
+  @Override
+  protected void setupConfigurableCapacities(CapacitySchedulerConfiguration
+      configuration) {
+    super.setupConfigurableCapacities(queueCapacities);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
new file mode 100644
index 0000000..aee6405
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -0,0 +1,745 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .queuemanagement;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .AbstractAutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .AutoCreatedLeafQueueConfig;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .AutoCreatedQueueManagementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ManagedParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .QueueManagementChange;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
+    .FiCaSchedulerApp;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager
+    .NO_LABEL;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+    .capacity.CSQueueUtils.EPSILON;
+
+/**
+ * Capacity Management policy for auto created leaf queues
+ * <p>
+ * Assigns capacity if available to leaf queues based on application
+ * submission order i.e leaf queues are assigned capacity in FCFS order based
+ * on application submission time.  Updates leaf queue capacities to 0 when
+ * there are no pending or running apps under that queue.
+ */
+public class GuaranteedOrZeroCapacityOverTimePolicy
+    implements AutoCreatedQueueManagementPolicy {
+
+  private CapacitySchedulerContext scheduler;
+  private ManagedParentQueue managedParentQueue;
+
+  private static final Log LOG = LogFactory.getLog(
+      GuaranteedOrZeroCapacityOverTimePolicy.class);
+
+  private AutoCreatedLeafQueueConfig ZERO_CAPACITY_ENTITLEMENT;
+
+  private ReentrantReadWriteLock.WriteLock writeLock;
+
+  private ReentrantReadWriteLock.ReadLock readLock;
+
+  private ParentQueueState parentQueueState = new ParentQueueState();
+
+  private AutoCreatedLeafQueueConfig leafQueueTemplate;
+
+  private QueueCapacities leafQueueTemplateCapacities;
+
+  private Map<String, LeafQueueState> leafQueueStateMap = new HashMap<>();
+
+  private Clock clock = new MonotonicClock();
+
+  private class LeafQueueState {
+
+    private AtomicBoolean isActive = new AtomicBoolean(false);
+
+    private long mostRecentActivationTime;
+
+    private long mostRecentDeactivationTime;
+
+    public long getMostRecentActivationTime() {
+      return mostRecentActivationTime;
+    }
+
+    public long getMostRecentDeactivationTime() {
+      return mostRecentDeactivationTime;
+    }
+
+    /**
+     * Is the queue currently active or deactivated?
+     *
+     * @return true if Active else false
+     */
+    public boolean isActive() {
+      return isActive.get();
+    }
+
+    private boolean activate() {
+      boolean ret = isActive.compareAndSet(false, true);
+      mostRecentActivationTime = clock.getTime();
+      return ret;
+    }
+
+    private boolean deactivate() {
+      boolean ret = isActive.compareAndSet(true, false);
+      mostRecentDeactivationTime = clock.getTime();
+      return ret;
+    }
+  }
+
+  private boolean containsLeafQueue(String leafQueueName) {
+    return leafQueueStateMap.containsKey(leafQueueName);
+  }
+
+  private boolean addLeafQueueStateIfNotExists(String leafQueueName,
+      LeafQueueState leafQueueState) {
+    if (!containsLeafQueue(leafQueueName)) {
+      leafQueueStateMap.put(leafQueueName, leafQueueState);
+      return true;
+    }
+    return false;
+  }
+
+  private boolean addLeafQueueStateIfNotExists(LeafQueue leafQueue) {
+    return addLeafQueueStateIfNotExists(leafQueue.getQueueName(),
+        new LeafQueueState());
+  }
+
+  private void clearLeafQueueState() {
+    leafQueueStateMap.clear();
+  }
+
+  private class ParentQueueState {
+
+    private Map<String, Float> totalAbsoluteActivatedChildQueueCapacityByLabel =
+        new HashMap<String, Float>();
+
+    private float getAbsoluteActivatedChildQueueCapacity() {
+      return getAbsoluteActivatedChildQueueCapacity(NO_LABEL);
+    }
+
+    private float getAbsoluteActivatedChildQueueCapacity(String nodeLabel) {
+      try {
+        readLock.lock();
+        Float totalActivatedCapacity = getByLabel(nodeLabel);
+        if (totalActivatedCapacity != null) {
+          return totalActivatedCapacity;
+        } else{
+          return 0;
+        }
+      } finally {
+        readLock.unlock();
+      }
+    }
+
+    private void incAbsoluteActivatedChildCapacity(String nodeLabel,
+        float childQueueCapacity) {
+      try {
+        writeLock.lock();
+        Float activatedChildCapacity = getByLabel(nodeLabel);
+        if (activatedChildCapacity != null) {
+          setByLabel(nodeLabel, activatedChildCapacity + childQueueCapacity);
+        } else{
+          setByLabel(nodeLabel, childQueueCapacity);
+        }
+      } finally {
+        writeLock.unlock();
+      }
+    }
+
+    private void decAbsoluteActivatedChildCapacity(String nodeLabel,
+        float childQueueCapacity) {
+      try {
+        writeLock.lock();
+        Float activatedChildCapacity = getByLabel(nodeLabel);
+        if (activatedChildCapacity != null) {
+          setByLabel(nodeLabel, activatedChildCapacity - childQueueCapacity);
+        } else{
+          setByLabel(nodeLabel, childQueueCapacity);
+        }
+      } finally {
+        writeLock.unlock();
+      }
+    }
+
+    Float getByLabel(String label) {
+      return totalAbsoluteActivatedChildQueueCapacityByLabel.get(label);
+    }
+
+    Float setByLabel(String label, float val) {
+      return totalAbsoluteActivatedChildQueueCapacityByLabel.put(label, val);
+    }
+
+    void clear() {
+      totalAbsoluteActivatedChildQueueCapacityByLabel.clear();
+    }
+  }
+
+  /**
+   * Comparator that orders applications by their submit time
+   */
+  private class PendingApplicationComparator
+      implements Comparator<FiCaSchedulerApp> {
+
+    @Override
+    public int compare(FiCaSchedulerApp app1, FiCaSchedulerApp app2) {
+      RMApp rmApp1 = scheduler.getRMContext().getRMApps().get(
+          app1.getApplicationId());
+      RMApp rmApp2 = scheduler.getRMContext().getRMApps().get(
+          app2.getApplicationId());
+      if (rmApp1 != null && rmApp2 != null) {
+        return Long.compare(rmApp1.getSubmitTime(), rmApp2.getSubmitTime());
+      } else if (rmApp1 != null) {
+        return -1;
+      } else if (rmApp2 != null) {
+        return 1;
+      } else{
+        return 0;
+      }
+    }
+  }
+
+  private PendingApplicationComparator applicationComparator =
+      new PendingApplicationComparator();
+
+  @Override
+  public void init(final CapacitySchedulerContext schedulerContext,
+      final ParentQueue parentQueue) {
+    this.scheduler = schedulerContext;
+
+    ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+    readLock = lock.readLock();
+    writeLock = lock.writeLock();
+
+    if (!(parentQueue instanceof ManagedParentQueue)) {
+      throw new IllegalArgumentException(
+          "Expected instance of type " + ManagedParentQueue.class);
+    }
+
+    this.managedParentQueue = (ManagedParentQueue) parentQueue;
+
+    initializeLeafQueueTemplate(this.managedParentQueue);
+
+    LOG.info(
+        "Initialized queue management policy for parent queue " + parentQueue
+            .getQueueName() + " with leaf queue template capacities : ["
+            + leafQueueTemplate.getQueueCapacities() + "]");
+  }
+
+  private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue) {
+    leafQueueTemplate = parentQueue.getLeafQueueTemplate();
+
+    leafQueueTemplateCapacities = leafQueueTemplate.getQueueCapacities();
+
+    ZERO_CAPACITY_ENTITLEMENT = buildTemplate(0.0f,
+        leafQueueTemplateCapacities.getMaximumCapacity());
+  }
+
+  @Override
+  public List<QueueManagementChange> computeQueueManagementChanges()
+      throws SchedulerDynamicEditException {
+
+    //TODO : Add support for node labels on leaf queue template configurations
+    //synch/add missing leaf queue(s) if any to state
+    updateLeafQueueState();
+
+    try {
+      readLock.lock();
+      List<QueueManagementChange> queueManagementChanges = new ArrayList<>();
+
+      // check if any leaf queues need to be deactivated based on pending
+      // applications and
+      float parentAbsoluteCapacity =
+          managedParentQueue.getQueueCapacities().getAbsoluteCapacity();
+
+      float leafQueueTemplateAbsoluteCapacity =
+          leafQueueTemplateCapacities.getAbsoluteCapacity();
+      Map<String, QueueCapacities> deactivatedLeafQueues =
+          deactivateLeafQueuesIfInActive(managedParentQueue, queueManagementChanges);
+
+      float deactivatedCapacity = getTotalDeactivatedCapacity(
+          deactivatedLeafQueues);
+
+      float sumOfChildQueueActivatedCapacity = parentQueueState.
+          getAbsoluteActivatedChildQueueCapacity();
+
+      //Check if we need to activate anything at all?
+      float availableCapacity = getAvailableCapacity(parentAbsoluteCapacity,
+          deactivatedCapacity, sumOfChildQueueActivatedCapacity);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(
+            "Parent queue : " + managedParentQueue.getQueueName() + " absCapacity = "
+                + parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = "
+                + leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = "
+                + deactivatedCapacity + " , absChildActivatedCapacity = "
+                + sumOfChildQueueActivatedCapacity + ", availableCapacity = "
+                + availableCapacity);
+      }
+
+      if (availableCapacity >= leafQueueTemplateAbsoluteCapacity) {
+        //sort applications across leaf queues by submit time
+        List<FiCaSchedulerApp> pendingApps = getSortedPendingApplications();
+
+        if (pendingApps.size() > 0) {
+          int maxLeafQueuesTobeActivated = getMaxLeavesToBeActivated(
+              availableCapacity, leafQueueTemplateAbsoluteCapacity,
+              pendingApps.size());
+
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Found " + maxLeafQueuesTobeActivated
+                + " leaf queues to be activated with " + pendingApps.size()
+                + " apps ");
+          }
+
+          LinkedHashSet<String> leafQueuesToBeActivated = getSortedLeafQueues(
+              pendingApps, maxLeafQueuesTobeActivated,
+              deactivatedLeafQueues.keySet());
+
+          //Compute entitlement changes for the identified leaf queues
+          // which is appended to the List of queueManagementChanges
+          computeQueueManagementChanges(leafQueuesToBeActivated,
+              queueManagementChanges, availableCapacity,
+              leafQueueTemplateAbsoluteCapacity);
+
+          if (LOG.isDebugEnabled()) {
+            if (leafQueuesToBeActivated.size() > 0) {
+              LOG.debug(
+                  "Activated leaf queues : [" + leafQueuesToBeActivated + "]");
+            }
+          }
+        }
+      }
+      return queueManagementChanges;
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private float getTotalDeactivatedCapacity(
+      Map<String, QueueCapacities> deactivatedLeafQueues) {
+    float deactivatedCapacity = 0;
+    for (Iterator<Map.Entry<String, QueueCapacities>> iterator =
+         deactivatedLeafQueues.entrySet().iterator(); iterator.hasNext(); ) {
+      Map.Entry<String, QueueCapacities> deactivatedQueueCapacity =
+          iterator.next();
+      deactivatedCapacity +=
+          deactivatedQueueCapacity.getValue().getAbsoluteCapacity();
+    }
+    return deactivatedCapacity;
+  }
+
+  @VisibleForTesting
+  void updateLeafQueueState() {
+    try {
+      writeLock.lock();
+      Set<String> newQueues = new HashSet<>();
+      for (CSQueue newQueue : managedParentQueue.getChildQueues()) {
+        if (newQueue instanceof LeafQueue) {
+          addLeafQueueStateIfNotExists((LeafQueue) newQueue);
+          newQueues.add(newQueue.getQueueName());
+        }
+      }
+
+      for (Iterator<Map.Entry<String, LeafQueueState>> itr =
+           leafQueueStateMap.entrySet().iterator(); itr.hasNext(); ) {
+        Map.Entry<String, LeafQueueState> e = itr.next();
+        String queueName = e.getKey();
+        if (!newQueues.contains(queueName)) {
+          itr.remove();
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private LinkedHashSet<String> getSortedLeafQueues(
+      final List<FiCaSchedulerApp> pendingApps, int leafQueuesNeeded,
+      Set<String> deactivatedQueues) throws SchedulerDynamicEditException {
+
+    LinkedHashSet<String> leafQueues = new LinkedHashSet<>(leafQueuesNeeded);
+    int ctr = 0;
+    for (FiCaSchedulerApp app : pendingApps) {
+
+      AutoCreatedLeafQueue leafQueue =
+          (AutoCreatedLeafQueue) app.getCSLeafQueue();
+      String leafQueueName = leafQueue.getQueueName();
+
+      //Check if leafQueue is not active already and has any pending apps
+      if (ctr < leafQueuesNeeded) {
+
+        if (!isActive(leafQueue)) {
+          if (!deactivatedQueues.contains(leafQueueName)) {
+            if (addLeafQueueIfNotExists(leafQueues, leafQueueName)) {
+              ctr++;
+            }
+          }
+        }
+      } else{
+        break;
+      }
+    }
+    return leafQueues;
+  }
+
+  private boolean addLeafQueueIfNotExists(Set<String> leafQueues,
+      String leafQueueName) {
+    boolean ret = false;
+    if (!leafQueues.contains(leafQueueName)) {
+      ret = leafQueues.add(leafQueueName);
+    }
+    return ret;
+  }
+
+  @VisibleForTesting
+  public boolean isActive(final AutoCreatedLeafQueue leafQueue)
+      throws SchedulerDynamicEditException {
+    try {
+      readLock.lock();
+      LeafQueueState leafQueueStatus = getLeafQueueState(leafQueue);
+      return leafQueueStatus.isActive();
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  private Map<String, QueueCapacities> deactivateLeafQueuesIfInActive(
+      ParentQueue parentQueue,
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException {
+    Map<String, QueueCapacities> deactivatedQueues = new HashMap<>();
+
+    for (CSQueue childQueue : parentQueue.getChildQueues()) {
+      AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue;
+
+      if (isActive(leafQueue) && !hasPendingApps(leafQueue)) {
+        queueManagementChanges.add(
+            new QueueManagementChange.UpdateQueue(leafQueue,
+                ZERO_CAPACITY_ENTITLEMENT));
+        deactivatedQueues.put(leafQueue.getQueueName(),
+            leafQueueTemplateCapacities);
+      } else{
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(" Leaf queue has pending applications :  " + leafQueue
+              .getNumApplications() + ".Skipping deactivation for "
+              + leafQueue);
+        }
+      }
+    }
+
+    if (LOG.isDebugEnabled()) {
+      if (deactivatedQueues.size() > 0) {
+        LOG.debug("Deactivated leaf queues : " + deactivatedQueues);
+      }
+    }
+    return deactivatedQueues;
+  }
+
+  private void computeQueueManagementChanges(
+      Set<String> leafQueuesToBeActivated,
+      List<QueueManagementChange> queueManagementChanges,
+      final float availableCapacity,
+      final float leafQueueTemplateAbsoluteCapacity) {
+
+    float curAvailableCapacity = availableCapacity;
+
+    for (String curLeafQueue : leafQueuesToBeActivated) {
+      // Activate queues if capacity is available
+      if (curAvailableCapacity >= leafQueueTemplateAbsoluteCapacity) {
+        AutoCreatedLeafQueue leafQueue =
+            (AutoCreatedLeafQueue) scheduler.getCapacitySchedulerQueueManager()
+                .getQueue(curLeafQueue);
+        if (leafQueue != null) {
+          AutoCreatedLeafQueueConfig newTemplate = buildTemplate(
+              leafQueueTemplateCapacities.getCapacity(),
+              leafQueueTemplateCapacities.getMaximumCapacity());
+          queueManagementChanges.add(
+              new QueueManagementChange.UpdateQueue(leafQueue, newTemplate));
+          curAvailableCapacity -= leafQueueTemplateAbsoluteCapacity;
+        } else{
+          LOG.warn(
+              "Could not find queue in scheduler while trying to deactivate "
+                  + curLeafQueue);
+        }
+      }
+    }
+  }
+
+  @VisibleForTesting
+  public int getMaxLeavesToBeActivated(float availableCapacity,
+      float childQueueAbsoluteCapacity, int numPendingApps)
+      throws SchedulerDynamicEditException {
+
+    if (childQueueAbsoluteCapacity > 0) {
+      int numLeafQueuesNeeded = (int) Math.floor(
+          availableCapacity / childQueueAbsoluteCapacity);
+
+      return Math.min(numLeafQueuesNeeded, numPendingApps);
+    } else{
+      throw new SchedulerDynamicEditException("Child queue absolute capacity "
+          + "is initialized to 0. Check parent queue's  " + managedParentQueue
+          .getQueueName() + " leaf queue template configuration");
+    }
+  }
+
+  private float getAvailableCapacity(float parentAbsCapacity,
+      float deactivatedAbsCapacity, float totalChildQueueActivatedCapacity) {
+    return parentAbsCapacity - totalChildQueueActivatedCapacity
+        + deactivatedAbsCapacity + EPSILON;
+  }
+
+  /**
+   * Commit queue management changes - which involves updating required state
+   * on parent/underlying leaf queues
+   *
+   * @param queueManagementChanges Queue Management changes to commit
+   * @throws SchedulerDynamicEditException when validation fails
+   */
+  @Override
+  public void commitQueueManagementChanges(
+      List<QueueManagementChange> queueManagementChanges)
+      throws SchedulerDynamicEditException {
+    try {
+      writeLock.lock();
+      for (QueueManagementChange queueManagementChange :
+          queueManagementChanges) {
+        AutoCreatedLeafQueueConfig updatedQueueTemplate =
+            queueManagementChange.getUpdatedQueueTemplate();
+        CSQueue queue = queueManagementChange.getQueue();
+        if (!(queue instanceof AutoCreatedLeafQueue)) {
+          throw new SchedulerDynamicEditException(
+              "Expected queue management change for AutoCreatedLeafQueue. "
+                  + "Found " + queue.getClass().getName());
+        }
+
+        AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) queue;
+
+        if (updatedQueueTemplate.getQueueCapacities().getCapacity() > 0) {
+          if (isActive(leafQueue)) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(
+                  "Queue is already active. Skipping activation : " + queue
+                      .getQueuePath());
+            }
+          } else{
+            activate(leafQueue);
+          }
+        } else{
+          if (!isActive(leafQueue)) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug(
+                  "Queue is already de-activated. " + "Skipping de-activation "
+                      + ": " + leafQueue.getQueuePath());
+            }
+          } else{
+            deactivate(leafQueue);
+          }
+        }
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void activate(final AutoCreatedLeafQueue leafQueue)
+      throws SchedulerDynamicEditException {
+    try {
+      writeLock.lock();
+      getLeafQueueState(leafQueue).activate();
+
+      parentQueueState.incAbsoluteActivatedChildCapacity(NO_LABEL,
+          leafQueueTemplateCapacities.getAbsoluteCapacity());
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  private void deactivate(final AutoCreatedLeafQueue leafQueue)
+      throws SchedulerDynamicEditException {
+    try {
+      writeLock.lock();
+      getLeafQueueState(leafQueue).deactivate();
+
+      for (String nodeLabel : managedParentQueue.getQueueCapacities()
+          .getExistingNodeLabels()) {
+        parentQueueState.decAbsoluteActivatedChildCapacity(nodeLabel,
+            leafQueueTemplateCapacities.getAbsoluteCapacity());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
+  public boolean hasPendingApps(final AutoCreatedLeafQueue leafQueue) {
+    return leafQueue.getNumApplications() > 0;
+  }
+
+  @Override
+  public void reinitialize(CapacitySchedulerContext schedulerContext,
+      final ParentQueue parentQueue) {
+    if (!(parentQueue instanceof ManagedParentQueue)) {
+      throw new IllegalStateException(
+          "Expected instance of type " + ManagedParentQueue.class + " found  "
+              + " : " + parentQueue.getClass());
+    }
+
+    if (this.managedParentQueue != null && !parentQueue.getQueuePath().equals(
+        this.managedParentQueue.getQueuePath())) {
+      throw new IllegalStateException(
+          "Expected parent queue path to match " + this.managedParentQueue
+              .getQueuePath() + " found : " + parentQueue.getQueuePath());
+    }
+
+    this.managedParentQueue = (ManagedParentQueue) parentQueue;
+
+    initializeLeafQueueTemplate(this.managedParentQueue);
+
+    //clear state
+    parentQueueState.clear();
+    clearLeafQueueState();
+
+    LOG.info(
+        "Reinitialized queue management policy for parent queue "
+            + parentQueue.getQueueName() +" with leaf queue template "
+            + "capacities : ["
+            + leafQueueTemplate.getQueueCapacities() + "]");
+  }
+
+  @Override
+  public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
+      AbstractAutoCreatedLeafQueue leafQueue)
+      throws SchedulerDynamicEditException {
+
+    if ( !(leafQueue instanceof  AutoCreatedLeafQueue)) {
+      throw new SchedulerDynamicEditException("Not an instance of "
+          + "AutoCreatedLeafQueue : " + leafQueue.getClass());
+    }
+
+    AutoCreatedLeafQueue autoCreatedLeafQueue =
+        (AutoCreatedLeafQueue) leafQueue;
+    AutoCreatedLeafQueueConfig template = ZERO_CAPACITY_ENTITLEMENT;
+    try {
+      writeLock.lock();
+      if (!addLeafQueueStateIfNotExists(leafQueue)) {
+        LOG.error("Leaf queue already exists in state : " + getLeafQueueState(
+            leafQueue));
+        throw new SchedulerDynamicEditException(
+            "Leaf queue already exists in state : " + getLeafQueueState(
+                leafQueue));
+      }
+
+      float availableCapacity = getAvailableCapacity(
+          managedParentQueue.getQueueCapacities().getAbsoluteCapacity(), 0,
+          parentQueueState.getAbsoluteActivatedChildQueueCapacity());
+
+      if (availableCapacity >= leafQueueTemplateCapacities
+          .getAbsoluteCapacity()) {
+        activate(autoCreatedLeafQueue);
+        template = buildTemplate(leafQueueTemplateCapacities.getCapacity(),
+            leafQueueTemplateCapacities.getMaximumCapacity());
+      }
+    } finally {
+      writeLock.unlock();
+    }
+    return template;
+  }
+
+  @VisibleForTesting
+  LeafQueueState getLeafQueueState(LeafQueue queue)
+      throws SchedulerDynamicEditException {
+    try {
+      readLock.lock();
+      String queueName = queue.getQueueName();
+      if (!containsLeafQueue(queueName)) {
+        throw new SchedulerDynamicEditException(
+            "Could not find leaf queue in " + "state " + queueName);
+      } else{
+        return leafQueueStateMap.get(queueName);
+      }
+    } finally {
+      readLock.unlock();
+    }
+  }
+
+  @VisibleForTesting
+  public float getAbsoluteActivatedChildQueueCapacity() {
+    return parentQueueState.getAbsoluteActivatedChildQueueCapacity();
+  }
+
+  private List<FiCaSchedulerApp> getSortedPendingApplications() {
+    List<FiCaSchedulerApp> apps = new ArrayList<>(
+        managedParentQueue.getAllApplications());
+    Collections.sort(apps, applicationComparator);
+    return apps;
+  }
+
+  private AutoCreatedLeafQueueConfig buildTemplate(float capacity,
+      float maxCapacity) {
+    AutoCreatedLeafQueueConfig.Builder templateBuilder =
+        new AutoCreatedLeafQueueConfig.Builder();
+
+    QueueCapacities capacities = new QueueCapacities(false);
+    templateBuilder.capacities(capacities);
+
+    for (String nodeLabel : managedParentQueue.getQueueCapacities()
+        .getExistingNodeLabels()) {
+      capacities.setCapacity(nodeLabel, capacity);
+      capacities.setMaximumCapacity(nodeLabel, maxCapacity);
+    }
+
+    return new AutoCreatedLeafQueueConfig(templateBuilder);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
index 2a751e3..f4182f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/QueueEntitlement.java
@@ -43,4 +43,26 @@ public class QueueEntitlement {
   public void setCapacity(float capacity) {
     this.capacity = capacity;
   }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o)
+      return true;
+    if (!(o instanceof QueueEntitlement))
+      return false;
+
+    QueueEntitlement that = (QueueEntitlement) o;
+
+    if (Float.compare(that.capacity, capacity) != 0)
+      return false;
+    return Float.compare(that.maxCapacity, maxCapacity) == 0;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = (capacity != +0.0f ? Float.floatToIntBits(capacity) : 0);
+    result = 31 * result + (maxCapacity != +0.0f ? Float.floatToIntBits(
+        maxCapacity) : 0);
+    return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/QueueManagementChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/QueueManagementChangeEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/QueueManagementChangeEvent.java
new file mode 100644
index 0000000..926e1be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/QueueManagementChangeEvent.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+    .QueueManagementChange;
+
+import java.util.List;
+
+/**
+ * Event to update scheduler of any queue management changes
+ */
+public class QueueManagementChangeEvent extends SchedulerEvent {
+
+  private ParentQueue parentQueue;
+  private List<QueueManagementChange> queueManagementChanges;
+
+  public QueueManagementChangeEvent(ParentQueue parentQueue,
+      List<QueueManagementChange> queueManagementChanges) {
+    super(SchedulerEventType.MANAGE_QUEUE);
+    this.parentQueue = parentQueue;
+    this.queueManagementChanges = queueManagementChanges;
+  }
+
+  public ParentQueue getParentQueue() {
+    return parentQueue;
+  }
+
+  public List<QueueManagementChange> getQueueManagementChanges() {
+    return queueManagementChanges;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 229e0bb..b107cf4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -51,5 +51,8 @@ public enum SchedulerEventType {
   MARK_CONTAINER_FOR_KILLABLE,
 
   // Cancel a killable container
-  MARK_CONTAINER_FOR_NONKILLABLE
+  MARK_CONTAINER_FOR_NONKILLABLE,
+
+  //Queue Management Change
+  MANAGE_QUEUE
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38643c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
deleted file mode 100644
index b403e72..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedLeafQueue.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for dynamic auto created leaf queues.
- * @see AutoCreatedLeafQueue
- */
-public class TestAutoCreatedLeafQueue {
-
-  private CapacitySchedulerConfiguration csConf;
-  private CapacitySchedulerContext csContext;
-  final static int DEF_MAX_APPS = 10000;
-  final static int GB = 1024;
-  private final ResourceCalculator resourceCalculator =
-      new DefaultResourceCalculator();
-  private AutoCreatedLeafQueue autoCreatedLeafQueue;
-
-  @Before
-  public void setup() throws IOException {
-    // setup a context / conf
-    csConf = new CapacitySchedulerConfiguration();
-    YarnConfiguration conf = new YarnConfiguration();
-    csContext = mock(CapacitySchedulerContext.class);
-    when(csContext.getConfiguration()).thenReturn(csConf);
-    when(csContext.getConf()).thenReturn(conf);
-    when(csContext.getMinimumResourceCapability()).thenReturn(
-        Resources.createResource(GB, 1));
-    when(csContext.getMaximumResourceCapability()).thenReturn(
-        Resources.createResource(16 * GB, 32));
-    when(csContext.getClusterResource()).thenReturn(
-        Resources.createResource(100 * 16 * GB, 100 * 32));
-    when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
-    RMContext mockRMContext = TestUtils.getMockRMContext();
-    when(csContext.getRMContext()).thenReturn(mockRMContext);
-
-    // create a queue
-    PlanQueue pq = new PlanQueue(csContext, "root", null, null);
-    autoCreatedLeafQueue = new AutoCreatedLeafQueue(csContext, "a", pq);
-  }
-
-  private void validateAutoCreatedLeafQueue(double capacity) {
-    assertTrue(" actual capacity: " + autoCreatedLeafQueue.getCapacity(),
-        autoCreatedLeafQueue.getCapacity() - capacity < CSQueueUtils.EPSILON);
-    assertEquals(autoCreatedLeafQueue.maxApplications, DEF_MAX_APPS);
-    assertEquals(autoCreatedLeafQueue.maxApplicationsPerUser, DEF_MAX_APPS);
-  }
-
-  @Test
-  public void testAddSubtractCapacity() throws Exception {
-
-    // verify that setting, adding, subtracting capacity works
-    autoCreatedLeafQueue.setCapacity(1.0F);
-    validateAutoCreatedLeafQueue(1);
-    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0.9f, 1f));
-    validateAutoCreatedLeafQueue(0.9);
-    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1f, 1f));
-    validateAutoCreatedLeafQueue(1);
-    autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(0f, 1f));
-    validateAutoCreatedLeafQueue(0);
-
-    try {
-      autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(1.1f, 1f));
-      fail();
-    } catch (SchedulerDynamicEditException iae) {
-      // expected
-      validateAutoCreatedLeafQueue(1);
-    }
-
-    try {
-      autoCreatedLeafQueue.setEntitlement(new QueueEntitlement(-0.1f, 1f));
-      fail();
-    } catch (SchedulerDynamicEditException iae) {
-      // expected
-      validateAutoCreatedLeafQueue(1);
-    }
-
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[19/50] hadoop git commit: YARN-7332. Compute effectiveCapacity per each resource vector. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7332. Compute effectiveCapacity per each resource vector. (Sunil G via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa3f6274
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa3f6274
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa3f6274

Branch: refs/heads/HDFS-7240
Commit: aa3f62740f71e6e5b2a424a9f4654c1a4ba1dbe6
Parents: d52627a
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Oct 27 10:16:33 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/ParentQueue.java         | 66 ++++++++++++--
 .../scheduler/capacity/TestParentQueue.java     | 94 ++++++++++++++++++++
 2 files changed, 153 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa3f6274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 5ab1494..940637e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.AccessType;
@@ -68,7 +69,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 @Private
@@ -928,24 +931,25 @@ public class ParentQueue extends AbstractCSQueue {
     // Factor to scale down effective resource: When cluster has sufficient
     // resources, effective_min_resources will be same as configured
     // min_resources.
-    float effectiveMinRatio = 1;
+    Resource numeratorForMinRatio = null;
     ResourceCalculator rc = this.csContext.getResourceCalculator();
     if (getQueueName().equals("root")) {
       if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
           clusterResource, resourceByLabel, configuredMinResources)) {
-        effectiveMinRatio = Resources.divide(rc, clusterResource,
-            resourceByLabel, configuredMinResources);
+        numeratorForMinRatio = resourceByLabel;
       }
     } else {
       if (Resources.lessThan(rc, clusterResource,
           queueResourceQuotas.getEffectiveMinResource(label),
           configuredMinResources)) {
-        effectiveMinRatio = Resources.divide(rc, clusterResource,
-            queueResourceQuotas.getEffectiveMinResource(label),
-            configuredMinResources);
+        numeratorForMinRatio = queueResourceQuotas
+            .getEffectiveMinResource(label);
       }
     }
 
+    Map<String, Float> effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
+        configuredMinResources, numeratorForMinRatio);
+
     // loop and do this for all child queues
     for (CSQueue childQueue : getChildQueues()) {
       Resource minResource = childQueue.getQueueResourceQuotas()
@@ -955,7 +959,8 @@ public class ParentQueue extends AbstractCSQueue {
       if (childQueue.getCapacityConfigType()
           .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
         childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
-            Resources.multiply(minResource, effectiveMinRatio));
+            getMinResourceNormalized(childQueue.getQueueName(), effectiveMinRatioPerResource,
+                minResource));
 
         // Max resource of a queue should be a minimum of {configuredMaxRes,
         // parentMaxRes}. parentMaxRes could be configured value. But if not
@@ -1003,6 +1008,53 @@ public class ParentQueue extends AbstractCSQueue {
     }
   }
 
+  private Resource getMinResourceNormalized(String name, Map<String, Float> effectiveMinRatio,
+      Resource minResource) {
+    Resource ret = Resource.newInstance(minResource);
+    int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+    for (int i = 0; i < maxLength; i++) {
+      ResourceInformation nResourceInformation = minResource
+          .getResourceInformation(i);
+
+      Float ratio = effectiveMinRatio.get(nResourceInformation.getName());
+      if (ratio != null) {
+        ret.setResourceValue(i,
+            (long) (nResourceInformation.getValue() * ratio.floatValue()));
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Updating min resource for Queue: " + name + " as "
+              + ret.getResourceInformation(i) + ", Actual resource: "
+              + nResourceInformation.getValue() + ", ratio: "
+              + ratio.floatValue());
+        }
+      }
+    }
+    return ret;
+  }
+
+  private Map<String, Float> getEffectiveMinRatioPerResource(
+      Resource configuredMinResources, Resource numeratorForMinRatio) {
+    Map<String, Float> effectiveMinRatioPerResource = new HashMap<>();
+    if (numeratorForMinRatio != null) {
+      int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+      for (int i = 0; i < maxLength; i++) {
+        ResourceInformation nResourceInformation = numeratorForMinRatio
+            .getResourceInformation(i);
+        ResourceInformation dResourceInformation = configuredMinResources
+            .getResourceInformation(i);
+
+        long nValue = nResourceInformation.getValue();
+        long dValue = UnitsConversionUtil.convert(
+            dResourceInformation.getUnits(), nResourceInformation.getUnits(),
+            dResourceInformation.getValue());
+        if (dValue != 0) {
+          effectiveMinRatioPerResource.put(nResourceInformation.getName(),
+              (float) nValue / dValue);
+        }
+      }
+    }
+    return effectiveMinRatioPerResource;
+  }
+
   private void deriveCapacityFromAbsoluteConfigurations(String label,
       Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa3f6274/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 25a9774..fe66aba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -68,6 +68,11 @@ import org.mockito.stubbing.Answer;
 
 public class TestParentQueue {
 
+  private static final Resource QUEUE_B_RESOURCE = Resource
+      .newInstance(14 * 1024, 22);
+  private static final Resource QUEUE_A_RESOURCE = Resource
+      .newInstance(6 * 1024, 10);
+
   private static final Log LOG = LogFactory.getLog(TestParentQueue.class);
   
   RMContext rmContext;
@@ -118,6 +123,23 @@ public class TestParentQueue {
     LOG.info("Setup top-level queues a and b");
   }
 
+  private void setupSingleLevelQueuesWithAbsoluteResource(
+      CapacitySchedulerConfiguration conf) {
+
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+
+    final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+    conf.setMinimumResourceRequirement("", Q_A,
+        QUEUE_A_RESOURCE);
+
+    final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+    conf.setMinimumResourceRequirement("", Q_B,
+        QUEUE_B_RESOURCE);
+
+    LOG.info("Setup top-level queues a and b with absolute resource");
+  }
+
   private FiCaSchedulerApp getMockApplication(int appId, String user) {
     FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
     doReturn(user).when(application).getUser();
@@ -931,6 +953,78 @@ public class TestParentQueue {
     reset(c);
   }
 
+  @Test
+  public void testAbsoluteResourceWithChangeInClusterResource()
+      throws Exception {
+    // Setup queue configs
+    setupSingleLevelQueuesWithAbsoluteResource(csConf);
+
+    Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+    CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf,
+        null, CapacitySchedulerConfiguration.ROOT, queues, queues,
+        TestUtils.spyHook);
+
+    // Setup some nodes
+    final int memoryPerNode = 10;
+    int coresPerNode = 16;
+    int numNodes = 2;
+
+    Resource clusterResource = Resources.createResource(
+        numNodes * (memoryPerNode * GB), numNodes * coresPerNode);
+    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
+    // Start testing
+    LeafQueue a = (LeafQueue) queues.get(A);
+    LeafQueue b = (LeafQueue) queues.get(B);
+
+    assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_A_RESOURCE);
+    assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_B_RESOURCE);
+    assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_A_RESOURCE);
+    assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_B_RESOURCE);
+
+    numNodes = 1;
+    clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+        numNodes * coresPerNode);
+    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
+    Resource QUEUE_B_RESOURCE_HALF = Resource.newInstance(7 * 1024, 11);
+    Resource QUEUE_A_RESOURCE_HALF = Resource.newInstance(3 * 1024, 5);
+    assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_A_RESOURCE);
+    assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_B_RESOURCE);
+    assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_A_RESOURCE_HALF);
+    assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_B_RESOURCE_HALF);
+
+    coresPerNode = 40;
+    clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+        numNodes * coresPerNode);
+    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
+    Resource QUEUE_B_RESOURCE_70PERC = Resource.newInstance(7 * 1024, 27);
+    Resource QUEUE_A_RESOURCE_30PERC = Resource.newInstance(3 * 1024, 12);
+    assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_A_RESOURCE);
+    assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+        QUEUE_B_RESOURCE);
+    assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_A_RESOURCE_30PERC);
+    assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+        QUEUE_B_RESOURCE_70PERC);
+  }
+
   @After
   public void tearDown() throws Exception {
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[16/50] hadoop git commit: YARN-7564. Cleanup to fix checkstyle issues of YARN-5881 branch. Contributed by Sunil G.

Posted by xy...@apache.org.
YARN-7564. Cleanup to fix checkstyle issues of YARN-5881 branch. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/daa1cdd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/daa1cdd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/daa1cdd0

Branch: refs/heads/HDFS-7240
Commit: daa1cdd062657a47acbf4b23f895860296241199
Parents: 1012b90
Author: Sunil G <su...@apache.org>
Authored: Tue Nov 28 14:07:09 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/util/StringUtils.java     | 18 ++++++-------
 .../impl/pb/QueueConfigurationsPBImpl.java      | 10 +++----
 .../resource/DominantResourceCalculator.java    |  2 +-
 .../yarn/util/resource/ResourceCalculator.java  | 12 ++++-----
 .../scheduler/AbstractResourceUsage.java        |  8 +++++-
 .../scheduler/ResourceUsage.java                |  6 -----
 .../scheduler/capacity/CSQueue.java             | 28 +++++++++++++++++---
 .../CapacitySchedulerConfiguration.java         | 28 ++++++++++++--------
 .../scheduler/capacity/TestReservations.java    |  3 +--
 9 files changed, 70 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 1be8a08..ebe7013 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1171,16 +1171,16 @@ public class StringUtils {
    * @return <code>true</code> if only contains letters, and is non-null
    */
   public static boolean isAlpha(String str) {
-      if (str == null) {
-          return false;
-      }
-      int sz = str.length();
-      for (int i = 0; i < sz; i++) {
-          if (Character.isLetter(str.charAt(i)) == false) {
-              return false;
-          }
+    if (str == null) {
+      return false;
+    }
+    int sz = str.length();
+    for (int i = 0; i < sz; i++) {
+      if (!Character.isLetter(str.charAt(i))) {
+        return false;
       }
-      return true;
+    }
+    return true;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index 80ef4b2..8266013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -236,12 +236,12 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
   }
 
   @Override
-  public void setConfiguredMinCapacity(Resource configuredMinResource) {
+  public void setConfiguredMinCapacity(Resource minResource) {
     maybeInitBuilder();
-    if (configuredMinResource == null) {
+    if (minResource == null) {
       builder.clearConfiguredMinCapacity();
     }
-    this.configuredMinResource = configuredMinResource;
+    this.configuredMinResource = minResource;
   }
 
   @Override
@@ -259,11 +259,11 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
   }
 
   @Override
-  public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+  public void setConfiguredMaxCapacity(Resource maxResource) {
     maybeInitBuilder();
     if (configuredMaxResource == null) {
       builder.clearConfiguredMaxCapacity();
     }
-    this.configuredMaxResource = configuredMaxResource;
+    this.configuredMaxResource = maxResource;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 26ddc18..6fed23b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -515,7 +515,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
     }
     return ret;
   }
-  
+
   @Override
   public Resource multiplyAndNormalizeUp(Resource r, double by,
       Resource stepFactor) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index dfd6684..1c42126 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -127,12 +127,12 @@ public abstract class ResourceCalculator {
       Resource r, double by, Resource stepFactor);
 
   /**
-   * Multiply resource <code>r</code> by factor <code>by</code> 
+   * Multiply resource <code>r</code> by factor <code>by</code>
    * and normalize up using step-factor <code>stepFactor</code>.
-   * 
+   *
    * @param r resource to be multiplied
    * @param by multiplier array for all resource types
-   * @param stepFactor factor by which to normalize up 
+   * @param stepFactor factor by which to normalize up
    * @return resulting normalized resource
    */
   public abstract Resource multiplyAndNormalizeUp(
@@ -149,7 +149,7 @@ public abstract class ResourceCalculator {
    */
   public abstract Resource multiplyAndNormalizeDown(
       Resource r, double by, Resource stepFactor);
-  
+
   /**
    * Normalize resource <code>r</code> given the base 
    * <code>minimumResource</code> and verify against max allowed
@@ -162,9 +162,7 @@ public abstract class ResourceCalculator {
    * @return normalized resource
    */
   public abstract Resource normalize(Resource r, Resource minimumResource,
-                                     Resource maximumResource, 
-                                     Resource stepFactor);
-
+      Resource maximumResource, Resource stepFactor);
 
   /**
    * Round-up resource <code>r</code> given factor <code>stepFactor</code>.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
index 3203543..1406fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -55,7 +55,10 @@ public class AbstractResourceUsage {
     usages.put(CommonNodeLabelsManager.NO_LABEL, noLabelUsages);
   }
 
-  // Usage enum here to make implement cleaner
+  /**
+   * Use enum here to make implementation more cleaner and readable.
+   * Indicates array index for each resource usage type.
+   */
   public enum ResourceType {
     // CACHED_USED and CACHED_PENDING may be read by anyone, but must only
     // be written by ordering policies
@@ -70,6 +73,9 @@ public class AbstractResourceUsage {
     }
   }
 
+  /**
+   * UsageByLabel stores resource array for all resource usage types.
+   */
   public static class UsageByLabel {
     // usage by label, contains all UsageType
     private final AtomicReferenceArray<Resource> resArr;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index ede4aec..711a468 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -18,13 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 624fcc7..6d79b6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -366,7 +366,7 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
   public QueueResourceQuotas getQueueResourceQuotas();
 
   /**
-   * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+   * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE.
    * @return CapacityConfigType
    */
   public CapacityConfigType getCapacityConfigType();
@@ -374,24 +374,46 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
   /**
    * Get effective capacity of queue. If min/max resource is configured,
    * preference will be given to absolute configuration over normal capacity.
-   * Also round down the result to normalizeDown.
    *
    * @param label
    *          partition
    * @return effective queue capacity
    */
   Resource getEffectiveCapacity(String label);
+
+  /**
+   * Get effective capacity of queue. If min/max resource is configured,
+   * preference will be given to absolute configuration over normal capacity.
+   * Also round down the result to normalizeDown.
+   *
+   * @param label
+   *          partition
+   * @param factor
+   *          factor to normalize down 
+   * @return effective queue capacity
+   */
   Resource getEffectiveCapacityDown(String label, Resource factor);
 
   /**
    * Get effective max capacity of queue. If min/max resource is configured,
    * preference will be given to absolute configuration over normal capacity.
-   * Also round down the result to normalizeDown.
    *
    * @param label
    *          partition
    * @return effective max queue capacity
    */
   Resource getEffectiveMaxCapacity(String label);
+
+  /**
+   * Get effective max capacity of queue. If min/max resource is configured,
+   * preference will be given to absolute configuration over normal capacity.
+   * Also round down the result to normalizeDown.
+   *
+   * @param label
+   *          partition
+   * @param factor
+   *          factor to normalize down 
+   * @return effective max queue capacity
+   */
   Resource getEffectiveMaxCapacityDown(String label, Resource factor);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index e47c8d0..a33d81a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
 import org.apache.hadoop.yarn.api.records.QueueState;
 import org.apache.hadoop.yarn.api.records.ReservationACL;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
@@ -320,7 +321,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   @Private
   public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
 
-  /** Configuring absolute min/max resources in a queue **/
+  /** Configuring absolute min/max resources in a queue. **/
   @Private
   public static final String MINIMUM_RESOURCE = "min-resource";
 
@@ -333,6 +334,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
 
   private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
 
+  /**
+   * Different resource types supported.
+   */
   public enum AbsoluteResourceType {
     MEMORY, VCORES;
   }
@@ -1825,7 +1829,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     }
 
     // Define resource here.
-    Resource resource = Resource.newInstance(0l, 0);
+    Resource resource = Resource.newInstance(0L, 0);
     Matcher matcher = RESOURCE_PATTERN.matcher(resourceString);
 
     /*
@@ -1852,7 +1856,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     }
 
     // Memory has to be configured always.
-    if (resource.getMemorySize() == 0l) {
+    if (resource.getMemorySize() == 0L) {
       return Resources.none();
     }
 
@@ -1884,14 +1888,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
     AbsoluteResourceType resType = AbsoluteResourceType
         .valueOf(StringUtils.toUpperCase(splits[0].trim()));
     switch (resType) {
-      case MEMORY :
-        resource.setMemorySize(resourceValue);
-        break;
-      case VCORES :
-        resource.setVirtualCores(resourceValue.intValue());
-        break;
-      default :
-        break;
+    case MEMORY :
+      resource.setMemorySize(resourceValue);
+      break;
+    case VCORES :
+      resource.setVirtualCores(resourceValue.intValue());
+      break;
+    default :
+      resource.setResourceInformation(splits[0].trim(), ResourceInformation
+          .newInstance(splits[0].trim(), units, resourceValue));
+      break;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 0855720..8705887 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -904,12 +904,11 @@ public class TestReservations {
     String host_1 = "host_1";
     FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
         8 * GB);
-    
+
     Resource clusterResource = Resources.createResource(2 * 8 * GB);
     root.updateClusterResource(clusterResource,
         new ResourceLimits(clusterResource));
 
-
     // Setup resource-requests
     Priority p = TestUtils.createMockPriority(5);
     SchedulerRequestKey priorityMap = toSchedulerKey(p);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[13/50] hadoop git commit: YARN-7483. CapacityScheduler test cases cleanup post YARN-5881. (Sunil G via wangda)

Posted by xy...@apache.org.
YARN-7483. CapacityScheduler test cases cleanup post YARN-5881. (Sunil G via wangda)

Change-Id: I9741a6baf5cb7352d05636efb6c0b24790e7589a


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7462c382
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7462c382
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7462c382

Branch: refs/heads/HDFS-7240
Commit: 7462c38277fa415703fc9074c1288d3bec73609c
Parents: e65ca92
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 16 11:35:48 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800

----------------------------------------------------------------------
 .../api/records/impl/pb/ResourcePBImpl.java     |  11 ++
 .../scheduler/capacity/TestLeafQueue.java       | 130 +++++++++++++++++--
 2 files changed, 131 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7462c382/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 4f90133..6ebed6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -151,6 +151,17 @@ public class ResourcePBImpl extends Resource {
         .newInstance(ResourceInformation.VCORES);
     this.setMemorySize(p.getMemory());
     this.setVirtualCores(p.getVirtualCores());
+
+    // Update missing resource information on respective index.
+    updateResourceInformationMap(types);
+  }
+
+  private void updateResourceInformationMap(ResourceInformation[] types) {
+    for (int i = 0; i < types.length; i++) {
+      if (resources[i] == null) {
+        resources[i] = ResourceInformation.newInstance(types[i]);
+      }
+    }
   }
 
   private static ResourceInformation newDefaultInformation(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7462c382/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 30bff78..1426e88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -1027,6 +1027,8 @@ public class TestLeafQueue {
     Resource clusterResource =
         Resources.createResource(numNodes * (8*GB), numNodes * 16);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
 
     // Setup resource-requests
     // app_0 asks for 3 3-GB containers
@@ -1083,9 +1085,15 @@ public class TestLeafQueue {
         a.assignContainers(clusterResource, node_1,
         new ResourceLimits(clusterResource),
         SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
-    assertEquals(12*GB, a.getUsedResources().getMemorySize());
-    assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
-    assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
+    assertEquals(9*GB, a.getUsedResources().getMemorySize());
+    assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
+    assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
+
+    assertEquals(4 * GB,
+        app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+
+    assertEquals(1 * GB,
+        app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
   }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1317,11 +1325,6 @@ public class TestLeafQueue {
     Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
     when(csContext.getNumClusterNodes()).thenReturn(numNodes);
 
-    ParentQueue root = (ParentQueue) queues
-        .get(CapacitySchedulerConfiguration.ROOT);
-    root.updateClusterResource(clusterResource,
-        new ResourceLimits(clusterResource));
-
     // Setup resource-requests
     Priority priority = TestUtils.createMockPriority(1);
     app_0.updateResourceRequests(Collections.singletonList(
@@ -1340,6 +1343,11 @@ public class TestLeafQueue {
     a.setUserLimit(50);
     a.setUserLimitFactor(2);
 
+    ParentQueue root = (ParentQueue) queues
+        .get(CapacitySchedulerConfiguration.ROOT);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
     // Now, only user_0 should be active since he is the only one with
     // outstanding requests
     assertEquals("There should only be 1 active user!", 
@@ -1368,8 +1376,8 @@ public class TestLeafQueue {
     assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
     assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
     assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
-    assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
-    
+    assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+
     // Submit requests for app_1 and set max-cap
     a.setMaxCapacity(.1f);
     root.updateClusterResource(clusterResource,
@@ -1405,6 +1413,108 @@ public class TestLeafQueue {
   }
 
   @Test
+  public void testUserHeadroomMultiApp() throws Exception {
+    // Mock the queue
+    LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
+    // unset maxCapacity
+    a.setMaxCapacity(1.0f);
+
+    // Users
+    final String user_0 = "user_0";
+    final String user_1 = "user_1";
+
+    // Submit applications
+    final ApplicationAttemptId appAttemptId_0 = TestUtils
+        .getMockApplicationAttemptId(0, 0);
+    FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
+        a.getAbstractUsersManager(), spyRMContext);
+    a.submitApplicationAttempt(app_0, user_0);
+
+    final ApplicationAttemptId appAttemptId_1 = TestUtils
+        .getMockApplicationAttemptId(1, 0);
+    FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
+        a.getAbstractUsersManager(), spyRMContext);
+    a.submitApplicationAttempt(app_1, user_0); // same user
+
+    final ApplicationAttemptId appAttemptId_2 = TestUtils
+        .getMockApplicationAttemptId(2, 0);
+    FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a,
+        a.getAbstractUsersManager(), spyRMContext);
+    a.submitApplicationAttempt(app_2, user_1);
+
+    // Setup some nodes
+    String host_0 = "127.0.0.1";
+    FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+        16 * GB);
+    String host_1 = "127.0.0.2";
+    FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
+        16 * GB);
+
+    Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
+        app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
+        app_1, app_2.getApplicationAttemptId(), app_2);
+    Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
+        node_0, node_1.getNodeID(), node_1);
+
+    final int numNodes = 2;
+    Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
+        1);
+    when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+    root.updateClusterResource(clusterResource,
+        new ResourceLimits(clusterResource));
+
+    Priority priority = TestUtils.createMockPriority(1);
+
+    app_0.updateResourceRequests(
+        Collections.singletonList(TestUtils.createResourceRequest(
+            ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory)));
+
+    applyCSAssignment(clusterResource,
+        a.assignContainers(clusterResource, node_0,
+            new ResourceLimits(clusterResource),
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+        a, nodes, apps);
+    assertEquals(1 * GB, a.getUsedResources().getMemorySize());
+    assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize());
+    assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize());
+    // Now, headroom is the same for all apps for a given user + queue combo
+    // and a change to any app's headroom is reflected for all the user's apps
+    // once those apps are active/have themselves calculated headroom for
+    // allocation at least one time
+    assertEquals(2 * GB, app_0.getHeadroom().getMemorySize());
+    assertEquals(2 * GB, app_1.getHeadroom().getMemorySize());// not yet active
+    assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active
+
+    app_1.updateResourceRequests(
+        Collections.singletonList(TestUtils.createResourceRequest(
+            ResourceRequest.ANY, 1 * GB, 2, true, priority, recordFactory)));
+
+    applyCSAssignment(clusterResource,
+        a.assignContainers(clusterResource, node_0,
+            new ResourceLimits(clusterResource),
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+        a, nodes, apps);
+    assertEquals(2 * GB, a.getUsedResources().getMemorySize());
+    assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize());
+    assertEquals(1 * GB, app_1.getCurrentConsumption().getMemorySize());
+    assertEquals(1 * GB, app_0.getHeadroom().getMemorySize());
+    assertEquals(1 * GB, app_1.getHeadroom().getMemorySize());// now active
+    assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active
+
+    // Complete container and verify that headroom is updated, for both apps
+    // for the user
+    RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
+    a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+        ContainerStatus.newInstance(rmContainer.getContainerId(),
+            ContainerState.COMPLETE, "",
+            ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+        RMContainerEventType.KILL, null, true);
+
+    assertEquals(2 * GB, app_0.getHeadroom().getMemorySize());
+    assertEquals(2 * GB, app_1.getHeadroom().getMemorySize());
+  }
+
+  @Test
   public void testSingleQueueWithMultipleUsers() throws Exception {
     
     // Mock the queue


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[25/50] hadoop git commit: HADOOP-15104. AliyunOSS: change the default value of max error retry. Contributed by Jinhu Wu

Posted by xy...@apache.org.
HADOOP-15104. AliyunOSS: change the default value of max error retry. Contributed by Jinhu Wu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce04340e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce04340e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce04340e

Branch: refs/heads/HDFS-7240
Commit: ce04340ec73617daff74378056a95c5d0cc0a790
Parents: 19e0894
Author: Kai Zheng <zh...@alibaba-inc.com>
Authored: Fri Dec 8 23:03:54 2017 +0800
Committer: Kai Zheng <zh...@alibaba-inc.com>
Committed: Fri Dec 8 23:03:54 2017 +0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce04340e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index baa171f..dd71842 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -66,7 +66,7 @@ public final class Constants {
 
   // Number of times we should retry errors
   public static final String MAX_ERROR_RETRIES_KEY = "fs.oss.attempts.maximum";
-  public static final int MAX_ERROR_RETRIES_DEFAULT = 20;
+  public static final int MAX_ERROR_RETRIES_DEFAULT = 10;
 
   // Time until we give up trying to establish a connection to oss
   public static final String ESTABLISH_TIMEOUT_KEY =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org