You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2018/12/15 00:04:32 UTC

[01/50] [abbrv] hadoop git commit: HADOOP-15899. Update AWS Java SDK versions in NOTICE.txt.

Repository: hadoop
Updated Branches:
  refs/heads/branch-2-jhung-test 04be9328f -> 5b580aaa7


HADOOP-15899. Update AWS Java SDK versions in NOTICE.txt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/035f7102
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/035f7102
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/035f7102

Branch: refs/heads/branch-2-jhung-test
Commit: 035f7102ef9a1f9ca5d15489b04a538e44ff6922
Parents: 85c2143
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Nov 5 15:32:54 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 NOTICE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/035f7102/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 33c12ed..6ab0214 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -17,7 +17,7 @@ which has the following notices:
 * This product includes software developed by IBM Corporation and others.
 
 The binary distribution of this product bundles binaries of
-AWS Java SDK 1.10.6,
+AWS Java SDK 1.11.199,
 which has the following notices:
  * This software includes third party software subject to the following
  copyrights: - XML parsing and utility functions from JetS3t - Copyright


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[34/50] [abbrv] hadoop git commit: YARN-7794. SLSRunner is not loading timeline service jars, causing failure.

Posted by jh...@apache.org.
YARN-7794. SLSRunner is not loading timeline service jars, causing failure.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15ce8df3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15ce8df3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15ce8df3

Branch: refs/heads/branch-2-jhung-test
Commit: 15ce8df30213ae2380a021899aa3a34027cffc8d
Parents: f9ea6f3
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Nov 30 17:03:01 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ce8df3/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 4a10938..0d90ded 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -86,7 +86,8 @@ calculateClasspath() {
   HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
   . $HADOOP_LIBEXEC_DIR/hadoop-config.sh
   SLS_HTML="${HADOOP_PREFIX}/share/hadoop/tools/sls/html"
-  export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${TOOL_PATH}:${SLS_HTML}"
+  TIMELINE_SERVICE="${HADOOP_YARN_HOME}/${YARN_DIR}/timelineservice/*"
+  export HADOOP_CLASSPATH="${HADOOP_CLASSPATH}:${TOOL_PATH}:${SLS_HTML}:${TIMELINE_SERVICE}"
 }
 ###############################################################################
 runSimulation() {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[24/50] [abbrv] hadoop git commit: Try something

Posted by jh...@apache.org.
Try something


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b580aaa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b580aaa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b580aaa

Branch: refs/heads/branch-2-jhung-test
Commit: 5b580aaa7e16048b162838d77bb48c5e9fddbaba
Parents: 5c02975
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Dec 14 16:02:55 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../apache/hadoop/cli/util/CommandExecutor.java |  8 +-
 .../apache/hadoop/crypto/key/TestKeyShell.java  |  8 +-
 .../apache/hadoop/fs/TestFsShellReturnCode.java | 16 ++--
 .../java/org/apache/hadoop/fs/TestTrash.java    |  8 +-
 .../hadoop/fs/shell/TestXAttrCommands.java      |  4 +-
 .../hadoop/ipc/TestDecayRpcScheduler.java       |  4 +-
 .../hadoop/security/alias/TestCredShell.java    |  4 +-
 .../apache/hadoop/test/GenericTestUtils.java    |  4 +-
 .../org/apache/hadoop/util/TestClasspath.java   |  8 +-
 .../hadoop/util/TestNativeLibraryChecker.java   |  4 +-
 .../crypto/key/kms/server/TestKMSAudit.java     |  4 +-
 .../federation/router/TestRouterAdminCLI.java   | 18 ++--
 .../org/apache/hadoop/hdfs/DFSTestUtil.java     |  8 +-
 .../hadoop/hdfs/TestBalancerBandwidth.java      |  4 +-
 .../org/apache/hadoop/hdfs/TestDFSShell.java    | 88 ++++++++++----------
 .../hadoop/hdfs/TestFsShellPermission.java      |  4 +-
 .../hadoop/hdfs/TestMaintenanceState.java       |  4 +-
 .../java/org/apache/hadoop/hdfs/TestQuota.java  | 12 +--
 .../hdfs/server/namenode/TestClusterId.java     | 12 +--
 .../namenode/TestDecommissioningStatus.java     |  4 +-
 .../namenode/TestMetadataVersionOutput.java     |  4 +-
 .../namenode/snapshot/TestSnapshotDeletion.java |  8 +-
 .../snapshot/TestSnapshotFileLength.java        |  8 +-
 .../namenode/snapshot/TestSnapshotRename.java   |  8 +-
 .../apache/hadoop/hdfs/tools/TestDFSAdmin.java  |  8 +-
 .../hadoop/hdfs/tools/TestDFSAdminWithHA.java   |  8 +-
 .../hadoop/hdfs/tools/TestDebugAdmin.java       |  8 +-
 .../TestOfflineEditsViewer.java                 |  4 +-
 .../TestOfflineImageViewer.java                 | 12 +--
 .../org/apache/hadoop/tools/TestJMXGet.java     |  4 +-
 .../java/org/apache/hadoop/tools/TestTools.java |  8 +-
 .../apache/hadoop/tracing/TestTraceAdmin.java   |  8 +-
 .../mapreduce/v2/hs/TestJobHistoryParsing.java  |  4 +-
 .../apache/hadoop/mapred/TestNetworkedJob.java  | 10 +--
 .../mapred/pipes/TestPipeApplication.java       |  6 +-
 .../hadoop/mapreduce/TestMRJobClient.java       |  8 +-
 .../java/org/apache/hadoop/ant/DfsTask.java     |  8 +-
 .../apache/hadoop/tools/TestHadoopArchives.java | 12 +--
 .../hadoop/tools/TestDistCpSyncReverseBase.java |  8 +-
 .../apache/hadoop/tools/TestDistCpSystem.java   |  4 +-
 .../org/apache/hadoop/tools/TestCopyFiles.java  |  4 +-
 .../org/apache/hadoop/tools/TestDistCh.java     |  8 +-
 .../mapred/gridmix/TestGridmixSubmission.java   |  4 +-
 .../hadoop/streaming/TestDumpTypedBytes.java    |  4 +-
 .../org/apache/hadoop/streaming/UtilTest.java   |  4 +-
 .../hadoop/yarn/client/cli/TestClusterCLI.java  |  2 +-
 .../hadoop/yarn/client/cli/TestLogsCLI.java     |  4 +-
 .../hadoop/yarn/client/cli/TestRMAdminCLI.java  | 16 ++--
 .../yarn/client/cli/TestSchedConfCLI.java       |  4 +-
 .../hadoop/yarn/client/cli/TestYarnCLI.java     |  2 +-
 .../TestLogAggregationIndexFileController.java  |  4 +-
 .../hadoop/registry/cli/TestRegistryCli.java    |  2 +-
 52 files changed, 216 insertions(+), 216 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
index 5ef129c..2c689df 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java
@@ -71,8 +71,8 @@ public abstract class CommandExecutor {
     PrintStream origOut = System.out;
     PrintStream origErr = System.err;
     
-    System.setOut(new PrintStream(bao));
-    System.setErr(new PrintStream(bao));
+    //System.setOut(new PrintStream(bao));
+    //System.setErr(new PrintStream(bao));
     
     try {
       exitCode = execute(cmd);
@@ -81,8 +81,8 @@ public abstract class CommandExecutor {
       lastException = e;
       exitCode = -1;
     } finally {
-      System.setOut(origOut);
-      System.setErr(origErr);
+      //System.setOut(origOut);
+      //System.setErr(origErr);
     }
     return new Result(bao.toString(), exitCode, lastException, cmd);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
index e2dcde5..55f8be8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
@@ -58,14 +58,14 @@ public class TestKeyShell {
     jceksProvider = "jceks://file" + jksPath.toUri();
     initialStdOut = System.out;
     initialStdErr = System.err;
-    System.setOut(new PrintStream(outContent));
-    System.setErr(new PrintStream(errContent));
+    //System.setOut(new PrintStream(outContent));
+    //System.setErr(new PrintStream(errContent));
   }
 
   @After
   public void cleanUp() throws Exception {
-    System.setOut(initialStdOut);
-    System.setErr(initialStdErr);
+    //System.setOut(initialStdOut);
+    //System.setErr(initialStdErr);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
index 024a076..62a0297 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
@@ -291,7 +291,7 @@ public class TestFsShellReturnCode {
     final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldErr = System.err;
-    System.setErr(out);
+    //System.setErr(out);
     final String results;
     try {
       Path tdir = new Path(TEST_ROOT_DIR, "notNullCopy");
@@ -310,7 +310,7 @@ public class TestFsShellReturnCode {
       assertTrue(" Not displaying the intended message ",results.contains("get: `"+args[1]+"': No such file or directory"));
     } finally {
       IOUtils.closeStream(out);
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
   }
   
@@ -322,7 +322,7 @@ public class TestFsShellReturnCode {
     final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream err = new PrintStream(bytes);
     final PrintStream oldErr = System.err;
-    System.setErr(err);
+    //System.setErr(err);
     final String results;
     try {
       int exit = shell.run(new String[]{"-rm", "nomatch*"});
@@ -331,7 +331,7 @@ public class TestFsShellReturnCode {
       assertTrue(results.contains("rm: `nomatch*': No such file or directory"));
     } finally {
       IOUtils.closeStream(err);
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
   }
 
@@ -343,14 +343,14 @@ public class TestFsShellReturnCode {
     final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream err = new PrintStream(bytes);
     final PrintStream oldErr = System.err;
-    System.setErr(err);
+    //System.setErr(err);
     try {
       int exit = shell.run(new String[]{"-rm", "-f", "nomatch*"});
       assertEquals(0, exit);
       assertTrue(bytes.toString().isEmpty());
     } finally {
       IOUtils.closeStream(err);
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
   }
 
@@ -371,7 +371,7 @@ public class TestFsShellReturnCode {
     final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldErr = System.err;
-    System.setErr(out);
+    //System.setErr(out);
     final String results;
     try {
       int run = shell.run(args);
@@ -380,7 +380,7 @@ public class TestFsShellReturnCode {
       assertTrue("Return code should be 0", run == 0);
     } finally {
       IOUtils.closeStream(out);
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
     
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 1a6d580..5f9971c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -435,8 +435,8 @@ public class TestTrash extends TestCase {
       PrintStream stderr = System.err;
       ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
       PrintStream newOut = new PrintStream(byteStream);
-      System.setOut(newOut);
-      System.setErr(newOut);
+      //System.setOut(newOut);
+      //System.setErr(newOut);
       try {
         shell.run(args);
       } catch (Exception e) {
@@ -444,8 +444,8 @@ public class TestTrash extends TestCase {
             e.getLocalizedMessage());
       }
       String output = byteStream.toString();
-      System.setOut(stdout);
-      System.setErr(stderr);
+      //System.setOut(stdout);
+      //System.setErr(stderr);
       assertTrue("skipTrash wasn't suggested as remedy to failed rm command" +
           " or we deleted / even though we could not get server defaults",
           output.indexOf("Consider using -skipTrash option") != -1 ||

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestXAttrCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestXAttrCommands.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestXAttrCommands.java
index af0a2c3..2a4b95f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestXAttrCommands.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestXAttrCommands.java
@@ -41,14 +41,14 @@ public class TestXAttrCommands {
   public void setup() throws IOException {
     errContent.reset();
     initialStdErr = System.err;
-    System.setErr(new PrintStream(errContent));
+    //System.setErr(new PrintStream(errContent));
     conf = new Configuration();
   }
   
   @After
   public void cleanUp() throws Exception {
     errContent.reset();
-    System.setErr(initialStdErr);
+    //System.setErr(initialStdErr);
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
index 10ab40a..dba1e24 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestDecayRpcScheduler.java
@@ -259,7 +259,7 @@ public class TestDecayRpcScheduler {
     PrintStream output = System.out;
     try {
       ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
+      //System.setOut(new PrintStream(bytes));
 
       // initializing DefaultMetricsSystem here would set "monitoring" flag in
       // MetricsSystemImpl to true
@@ -270,7 +270,7 @@ public class TestDecayRpcScheduler {
       assertFalse(bytes.toString().contains("NullPointerException"));
     } finally {
       //set systout back
-      System.setOut(output);
+      //System.setOut(output);
     }
 
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index 37243dd..0159a35 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -45,8 +45,8 @@ public class TestCredShell {
 
   @Before
   public void setup() throws Exception {
-    System.setOut(new PrintStream(outContent));
-    System.setErr(new PrintStream(errContent));
+    //System.setOut(new PrintStream(outContent));
+    //System.setErr(new PrintStream(errContent));
     final Path jksPath = new Path(tmpDir.toString(), "keystore.jceks");
     new File(jksPath.toString()).delete();
     jceksProvider = "jceks://file" + jksPath.toUri();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 029429b..6b8a64c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -427,7 +427,7 @@ public abstract class GenericTestUtils {
       bytes = new ByteArrayOutputStream();
       bytesPrintStream = new PrintStream(bytes);
       oldErr = System.err;
-      System.setErr(new TeePrintStream(oldErr, bytesPrintStream));
+      //System.setErr(new TeePrintStream(oldErr, bytesPrintStream));
     }
 
     public String getOutput() {
@@ -437,7 +437,7 @@ public abstract class GenericTestUtils {
     @Override
     public void close() throws Exception {
       IOUtils.closeQuietly(bytesPrintStream);
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
index e6a6cb1..4c445ff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClasspath.java
@@ -65,17 +65,17 @@ public class TestClasspath {
 
     stdout = new ByteArrayOutputStream();
     printStdout = new PrintStream(stdout);
-    System.setOut(printStdout);
+    //System.setOut(printStdout);
 
     stderr = new ByteArrayOutputStream();
     printStderr = new PrintStream(stderr);
-    System.setErr(printStderr);
+    //System.setErr(printStderr);
   }
 
   @After
   public void tearDown() {
-    System.setOut(oldStdout);
-    System.setErr(oldStderr);
+    //System.setOut(oldStdout);
+    //System.setErr(oldStderr);
     IOUtils.cleanupWithLogger(LOG, printStdout, printStderr);
     assertTrue(FileUtil.fullyDelete(TEST_DIR));
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
index 7589e5a..3a9b69e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeLibraryChecker.java
@@ -65,7 +65,7 @@ public class TestNativeLibraryChecker extends TestCase {
     ExitUtil.disableSystemExit();
     ByteArrayOutputStream outContent = new ByteArrayOutputStream();
     PrintStream originalPs = System.out;
-    System.setOut(new PrintStream(outContent));
+    //System.setOut(new PrintStream(outContent));
     try {
       NativeLibraryChecker.main(args);
     } catch (ExitException e) {
@@ -77,7 +77,7 @@ public class TestNativeLibraryChecker extends TestCase {
       if (NativeCodeLoader.isNativeCodeLoaded()) {
         assertEquals(outContent.toString().indexOf("hadoop:  true") != -1, true);
       }
-      System.setOut(originalPs);
+      //System.setOut(originalPs);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
index 5955c61..16989f0 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -66,7 +66,7 @@ public class TestKMSAudit {
     memOut = new ByteArrayOutputStream();
     filterOut = new FilterOut(memOut);
     capturedOut = new PrintStream(filterOut);
-    System.setErr(capturedOut);
+    //System.setErr(capturedOut);
     PropertyConfigurator.configure(Thread.currentThread().
         getContextClassLoader()
         .getResourceAsStream("log4j-kmsaudit.properties"));
@@ -76,7 +76,7 @@ public class TestKMSAudit {
 
   @After
   public void cleanUp() {
-    System.setErr(originalOut);
+    //System.setErr(originalOut);
     LogManager.resetConfiguration();
     kmsAudit.shutdown();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 98a77f8..7102185 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
@@ -136,8 +136,8 @@ public class TestRouterAdminCLI {
   @After
   public void tearDown() {
     // set back system out/err
-    System.setOut(OLD_OUT);
-    System.setErr(OLD_ERR);
+    //System.setOut(OLD_OUT);
+    //System.setErr(OLD_ERR);
   }
 
   @Test
@@ -257,7 +257,7 @@ public class TestRouterAdminCLI {
     assertEquals(0, ToolRunner.run(admin, argv));
 
     // re-set system out for testing
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     stateStore.loadCache(MountTableStoreImpl.class, true);
     argv = new String[] {"-ls", src};
     assertEquals(0, ToolRunner.run(admin, argv));
@@ -313,7 +313,7 @@ public class TestRouterAdminCLI {
 
     // remove an invalid mount table
     String invalidPath = "/invalid";
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     argv = new String[] {"-rm", invalidPath};
     assertEquals(0, ToolRunner.run(admin, argv));
     assertTrue(out.toString().contains(
@@ -370,7 +370,7 @@ public class TestRouterAdminCLI {
   @Test
   public void testMountTablePermissions() throws Exception {
     // re-set system out for testing
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     // use superuser to add new mount table with only read permission
     String[] argv = new String[] {"-add", "/testpath2-1", "ns0", "/testdir2-1",
         "-owner", TEST_USER, "-group", TEST_USER, "-mode", "0455"};
@@ -519,7 +519,7 @@ public class TestRouterAdminCLI {
         routerContext.getRouter().getRouterState());
     assertTrue(routerContext.getRouter().getSafemodeService().isInSafeMode());
 
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     assertEquals(0, ToolRunner.run(admin,
         new String[] {"-safemode", "get"}));
     assertTrue(out.toString().contains("true"));
@@ -575,7 +575,7 @@ public class TestRouterAdminCLI {
         new String[] {"-nameservice", "disable", "ns0"}));
 
     stateStore.loadCache(DisabledNameserviceStoreImpl.class, true);
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     assertEquals(0, ToolRunner.run(admin,
         new String[] {"-getDisabledNameservices"}));
     assertTrue("ns0 should be reported: " + out,
@@ -593,7 +593,7 @@ public class TestRouterAdminCLI {
         out.toString().contains("ns0"));
 
     // Wrong commands
-    System.setErr(new PrintStream(err));
+    //System.setErr(new PrintStream(err));
     assertEquals(-1, ToolRunner.run(admin,
         new String[] {"-nameservice", "enable"}));
     String msg = "Not enough parameters specificed for cmd -nameservice";
@@ -624,7 +624,7 @@ public class TestRouterAdminCLI {
 
   @Test
   public void testUpdateNonExistingMountTable() throws Exception {
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     String nsId = "ns0";
     String src = "/test-updateNonExistingMounttable";
     String dest = "/updateNonExistingMounttable";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index e690c6d..088cf59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1706,16 +1706,16 @@ public class DFSTestUtil {
     try {
       ByteArrayOutputStream bs = new ByteArrayOutputStream(1024);
       try (PrintStream out = new PrintStream(bs)) {
-        System.setOut(out);
-        System.setErr(out);
+        //System.setOut(out);
+        //System.setErr(out);
         ret = tool.run(cmds);
         System.out.flush();
         System.err.flush();
       }
       output = bs.toString();
     } finally {
-      System.setOut(origOut);
-      System.setErr(origErr);
+      //System.setOut(origOut);
+      //System.setErr(origErr);
     }
     System.out.println("Output for command: " + cmd + " retcode: " + ret);
     if (output != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java
index 6bbe3a1..0190ca2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBalancerBandwidth.java
@@ -120,7 +120,7 @@ public class TestBalancerBandwidth {
     PrintStream initialStdOut = System.out;
     outContent.reset();
     try {
-      System.setOut(outStream);
+      //System.setOut(outStream);
       int exitCode = admin.run(args);
       assertEquals("DFSAdmin should return 0", 0, exitCode);
       String bandwidthOutMsg = "Balancer bandwidth is " + expectedBandwidth
@@ -128,7 +128,7 @@ public class TestBalancerBandwidth {
       String strOut = new String(outContent.toByteArray(), UTF8);
       assertTrue("Wrong balancer bandwidth!", strOut.contains(bandwidthOutMsg));
     } finally {
-      System.setOut(initialStdOut);
+      //System.setOut(initialStdOut);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index ce8694a..555b674 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -233,7 +233,7 @@ public class TestDFSShell {
     PrintStream psBackup = System.out;
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     PrintStream psOut = new PrintStream(out);
-    System.setOut(psOut);
+    //System.setOut(psOut);
     FsShell shell = new FsShell(dfs.getConf());
     try {
       Path myPath = new Path("/testDu/dir");
@@ -290,7 +290,7 @@ public class TestDFSShell {
       Long combinedLength = myFileLength + myFile2Length;
       assertThat(returnString, containsString(combinedLength.toString()));
     } finally {
-      System.setOut(psBackup);
+      //System.setOut(psBackup);
     }
                                   
   }
@@ -304,7 +304,7 @@ public class TestDFSShell {
     final FsShell shell = new FsShell(dfs.getConf());
 
     try {
-      System.setOut(psOut);
+      //System.setOut(psOut);
       final Path parent = new Path("/testDuSnapshots");
       final Path dir = new Path(parent, "dir");
       mkdir(dfs, dir);
@@ -407,7 +407,7 @@ public class TestDFSShell {
       assertTrue(returnString.contains(exludeSnapshotLength.toString()));
       out.reset();
     } finally {
-      System.setOut(psBackup);
+      //System.setOut(psBackup);
     }
   }
 
@@ -416,7 +416,7 @@ public class TestDFSShell {
     final PrintStream psBackup = System.out;
     final ByteArrayOutputStream out = new ByteArrayOutputStream();
     final PrintStream psOut = new PrintStream(out);
-    System.setOut(psOut);
+    //System.setOut(psOut);
     final FsShell shell = new FsShell(dfs.getConf());
 
     try {
@@ -491,7 +491,7 @@ public class TestDFSShell {
       assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
       out.reset();
     } finally {
-      System.setOut(psBackup);
+      //System.setOut(psBackup);
     }
   }
 
@@ -590,7 +590,7 @@ public class TestDFSShell {
       bak = System.err;
       ByteArrayOutputStream out = new ByteArrayOutputStream();
       PrintStream tmp = new PrintStream(out);
-      System.setErr(tmp);
+      //System.setErr(tmp);
       String[] argv = new String[2];
       argv[0] = "-cat";
       argv[1] = root.toUri().getPath();
@@ -719,7 +719,7 @@ public class TestDFSShell {
       assertTrue(returned.isEmpty());
     } finally {
       if (bak != null) {
-        System.setErr(bak);
+        //System.setErr(bak);
       }
     }
   }
@@ -860,7 +860,7 @@ public class TestDFSShell {
       pout.write(text.getBytes());
     }
     final ByteArrayOutputStream out = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     final String[] argv = new String[]{"-tail", testFile.toString()};
     final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv);
 
@@ -882,7 +882,7 @@ public class TestDFSShell {
     dfs.create(testFile);
 
     final ByteArrayOutputStream out = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
     final Thread tailer = new Thread() {
       @Override
       public void run() {
@@ -950,7 +950,7 @@ public class TestDFSShell {
       zout.close();
 
       ByteArrayOutputStream out = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
 
       String[] argv = new String[2];
       argv[0] = "-text";
@@ -970,7 +970,7 @@ public class TestDFSShell {
       writer.append(new Text("Foo"), new Text("Bar"));
       writer.close();
       out = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
       argv = new String[2];
       argv[0] = "-text";
       argv[1] = new Path(root, "file.gz").toString();
@@ -987,7 +987,7 @@ public class TestDFSShell {
       dout.write(outbytes);
       dout.close();
       out = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
       argv = new String[2];
       argv[0] = "-text";
       argv[1] = new Path(root, "file.deflate").toString();
@@ -1008,7 +1008,7 @@ public class TestDFSShell {
       fout.write(writebytes);
       fout.close();
       out = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
       argv = new String[2];
       argv[0] = "-text";
       argv[1] = new Path(root, p).toString();
@@ -1019,7 +1019,7 @@ public class TestDFSShell {
       out.reset();
     } finally {
       if (null != bak) {
-        System.setOut(bak);
+        //System.setOut(bak);
       }
     }
   }
@@ -1152,7 +1152,7 @@ public class TestDFSShell {
     ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 
     PrintStream out = new PrintStream(bytes);
     PrintStream oldOut = System.out;
-    System.setOut(out);
+    //System.setOut(out);
     Scanner in = null;
     String results = null;
     try {
@@ -1162,7 +1162,7 @@ public class TestDFSShell {
       assertEquals(dirs, in.nextLong());
       assertEquals(files, in.nextLong());
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       if (in!=null) in.close();
       IOUtils.closeStream(out);
       System.out.println("results:\n" + results);
@@ -1812,7 +1812,7 @@ public class TestDFSShell {
           FsShell fshell = new FsShell(dfs.getConf());
           ByteArrayOutputStream out = new ByteArrayOutputStream();
           PrintStream tmp = new PrintStream(out);
-          System.setErr(tmp);
+          //System.setErr(tmp);
           String[] args = new String[2];
           args[0] = "-ls";
           args[1] = "/foo";
@@ -1827,7 +1827,7 @@ public class TestDFSShell {
       });
     } finally {
       if (bak != null) {
-        System.setErr(bak);
+        //System.setErr(bak);
       }
     }
   }
@@ -1945,7 +1945,7 @@ public class TestDFSShell {
     final String atime2 = fmt.format(new Date(status2.getAccessTime()));
 
     final ByteArrayOutputStream out = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(out));
+    //System.setOut(new PrintStream(out));
 
     doFsStat(dfs.getConf(), null);
 
@@ -2043,15 +2043,15 @@ public class TestDFSShell {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
-    System.setOut(out);
-    System.setErr(out);
+    //System.setOut(out);
+    //System.setErr(out);
     final String results;
     try {
       assertEquals(returnvalue, shell.run(new String[]{"-lsr", root}));
       results = bytes.toString();
     } finally {
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
       IOUtils.closeStream(out);
     }
     System.out.println("results:\n" + results);
@@ -2662,8 +2662,8 @@ public class TestDFSShell {
       final ByteArrayOutputStream bao = new ByteArrayOutputStream();
       final PrintStream capture = new PrintStream(bao);
 
-      System.setOut(capture);
-      System.setErr(capture);
+      //System.setOut(capture);
+      //System.setErr(capture);
 
       final String[] argv = new String[] { "-setrep", "1", hdfsFile.toString() };
 
@@ -2671,8 +2671,8 @@ public class TestDFSShell {
         assertEquals("Command did not return the expected exit code",
             1, shell.run(argv));
       } finally {
-        System.setOut(origOut);
-        System.setErr(origErr);
+        //System.setOut(origOut);
+        //System.setErr(origErr);
       }
 
       assertTrue("Error message is not the expected error message"
@@ -2865,7 +2865,7 @@ public class TestDFSShell {
       bak = System.err;
       final FsShell fshell = new FsShell(dfs.getConf());
       final ByteArrayOutputStream out = new ByteArrayOutputStream();
-      System.setErr(new PrintStream(out));
+      //System.setErr(new PrintStream(out));
       
       // No permission to write xattr
       dfs.setPermission(p, new FsPermission((short) 0700));
@@ -2914,7 +2914,7 @@ public class TestDFSShell {
       });
     } finally {
       if (bak != null) {
-        System.setErr(bak);
+        //System.setErr(bak);
       }
     }
   }
@@ -2930,7 +2930,7 @@ public class TestDFSShell {
 
       final FsShell fshell = new FsShell(dfs.getConf());
       final ByteArrayOutputStream out = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
 
       doSetXattr(out, fshell,
         new String[] {"-setfattr", "-n", "User.Foo", p.toString()},
@@ -2977,7 +2977,7 @@ public class TestDFSShell {
 
     } finally {
       if (bak != null) {
-        System.setOut(bak);
+        //System.setOut(bak);
       }
     }
   }
@@ -3054,7 +3054,7 @@ public class TestDFSShell {
 
       final FsShell fshell = new FsShell(dfs.getConf());
       final ByteArrayOutputStream out = new ByteArrayOutputStream();
-      System.setErr(new PrintStream(out));
+      //System.setErr(new PrintStream(out));
 
       //Test 1.  Let user1 be owner for /foo
       user1.doAs(new PrivilegedExceptionAction<Object>() {
@@ -3196,7 +3196,7 @@ public class TestDFSShell {
       });
     } finally {
       if (bak != null) {
-        System.setErr(bak);
+        //System.setErr(bak);
       }
     }
   }
@@ -3219,7 +3219,7 @@ public class TestDFSShell {
 
       final FsShell fshell = new FsShell(dfs.getConf());
       final ByteArrayOutputStream out = new ByteArrayOutputStream();
-      System.setErr(new PrintStream(out));
+      //System.setErr(new PrintStream(out));
 
       // No permission for "other".
       dfs.setPermission(p, new FsPermission((short) 0700));
@@ -3256,7 +3256,7 @@ public class TestDFSShell {
       }
     } finally {
       if (bakErr != null) {
-        System.setErr(bakErr);
+        //System.setErr(bakErr);
       }
     }
   }
@@ -3325,7 +3325,7 @@ public class TestDFSShell {
     PrintStream syserr = System.err;
     final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(baos);
-    System.setErr(ps);
+    //System.setErr(ps);
     try {
       runCmd(shell, "-ls", "/.reserved");
       assertEquals(0, baos.toString().length());
@@ -3333,7 +3333,7 @@ public class TestDFSShell {
       runCmd(shell, "-ls", "/.reserved/raw/.reserved");
       assertTrue(baos.toString().contains("No such file or directory"));
     } finally {
-      System.setErr(syserr);
+      //System.setErr(syserr);
       cluster.shutdown();
     }
   }
@@ -3386,13 +3386,13 @@ public class TestDFSShell {
     PrintStream syserr = System.err;
     final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(baos);
-    System.setErr(ps);
+    //System.setErr(ps);
     try {
       FsShell shell = new FsShell(dfs.getConf());
       runCmd(shell, "-cp", src.toString(), "/.reserved");
       assertTrue(baos.toString().contains("Invalid path name /.reserved"));
     } finally {
-      System.setErr(syserr);
+      //System.setErr(syserr);
     }
   }
 
@@ -3402,13 +3402,13 @@ public class TestDFSShell {
     PrintStream syserr = System.err;
     final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(baos);
-    System.setErr(ps);
+    //System.setErr(ps);
     try {
       FsShell shell = new FsShell(dfs.getConf());
       runCmd(shell, "-chmod", "777", "/.reserved");
       assertTrue(baos.toString().contains("Invalid path name /.reserved"));
     } finally {
-      System.setErr(syserr);
+      //System.setErr(syserr);
     }
   }
 
@@ -3418,13 +3418,13 @@ public class TestDFSShell {
     PrintStream syserr = System.err;
     final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(baos);
-    System.setErr(ps);
+    //System.setErr(ps);
     try {
       FsShell shell = new FsShell(dfs.getConf());
       runCmd(shell, "-chown", "user1", "/.reserved");
       assertTrue(baos.toString().contains("Invalid path name /.reserved"));
     } finally {
-      System.setErr(syserr);
+      //System.setErr(syserr);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
index cc456b2..b54b15e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFsShellPermission.java
@@ -99,11 +99,11 @@ public class TestFsShellPermission {
     PrintStream old = System.out;
     int ret;
     try {
-      System.setOut(out);
+      //System.setOut(out);
       ret = shell.run(args);
       out.close();
     } finally {
-      System.setOut(old);
+      //System.setOut(old);
     }
     return String.valueOf(ret);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
index e0dfb4a..0c5ef0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
@@ -1139,8 +1139,8 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
   public void testReportMaintenanceNodes() throws Exception {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     ByteArrayOutputStream err = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(out));
-    System.setErr(new PrintStream(err));
+    //System.setOut(new PrintStream(out));
+    //System.setErr(new PrintStream(err));
 
     LOG.info("Starting testReportMaintenanceNodes");
     int expirationInMs = 30 * 1000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
index bf044e6..b13c6de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java
@@ -122,8 +122,8 @@ public class TestQuota {
   }
 
   private static void redirectStream() {
-    System.setOut(new PrintStream(OUT_STREAM));
-    System.setErr(new PrintStream(ERR_STREAM));
+    //System.setOut(new PrintStream(OUT_STREAM));
+    //System.setErr(new PrintStream(ERR_STREAM));
   }
 
   private static void resetStream() {
@@ -137,8 +137,8 @@ public class TestQuota {
       System.out.flush();
       System.err.flush();
     } finally {
-      System.setOut(OLD_OUT);
-      System.setErr(OLD_ERR);
+      //System.setOut(OLD_OUT);
+      //System.setErr(OLD_ERR);
     }
 
     if (cluster != null) {
@@ -1148,7 +1148,7 @@ public class TestQuota {
     ByteArrayOutputStream err = new ByteArrayOutputStream();
     PrintStream oldErr = System.err;
     try {
-      System.setErr(new PrintStream(err));
+      //System.setErr(new PrintStream(err));
       String[] args =
           { "-setSpaceQuota", "100", "-storageType", "COLD", "/testDir" };
       admin.run(args);
@@ -1156,7 +1156,7 @@ public class TestQuota {
       assertTrue(
           errOutput.contains(StorageType.getTypesSupportingQuota().toString()));
     } finally {
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
index d5a3948..549487e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
@@ -235,14 +235,14 @@ public class TestClusterId {
     PrintStream origErr = System.err;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream stdErr = new PrintStream(baos);
-    System.setErr(stdErr);
+    //System.setErr(stdErr);
     try {
       NameNode.createNameNode(argv, config);
 
       // Check if usage is printed
       assertTrue(baos.toString("UTF-8").contains("Usage: hdfs namenode"));
     } finally {
-      System.setErr(origErr);
+      //System.setErr(origErr);
     }
 
     // check if the version file does not exists.
@@ -263,14 +263,14 @@ public class TestClusterId {
     PrintStream origErr = System.err;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream stdErr = new PrintStream(baos);
-    System.setErr(stdErr);
+    //System.setErr(stdErr);
     try {
       NameNode.createNameNode(argv, config);
 
       // Check if usage is printed
       assertTrue(baos.toString("UTF-8").contains("Usage: hdfs namenode"));
     } finally {
-      System.setErr(origErr);
+      //System.setErr(origErr);
     }
 
     // check if the version file does not exists.
@@ -292,14 +292,14 @@ public class TestClusterId {
     PrintStream origErr = System.err;
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream stdErr = new PrintStream(baos);
-    System.setErr(stdErr);
+    //System.setErr(stdErr);
     try {
       NameNode.createNameNode(argv, config);
 
       // Check if usage is printed
       assertTrue(baos.toString("UTF-8").contains("Usage: hdfs namenode"));
     } finally {
-      System.setErr(origErr);
+      //System.setErr(origErr);
     }
 
     // check if the version file does not exists.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
index cbb7970..7585945 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
@@ -181,7 +181,7 @@ public class TestDecommissioningStatus {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(baos);
     PrintStream oldOut = System.out;
-    System.setOut(ps);
+    //System.setOut(ps);
     try {
       // Parse DFSAdmin just to check the count
       admin.report(new String[] {"-decommissioning"}, 0);
@@ -216,7 +216,7 @@ public class TestDecommissioningStatus {
             decomming.contains(id));
       }
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java
index e1663e2..e7c71f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java
@@ -76,7 +76,7 @@ public class TestMetadataVersionOutput {
     final ByteArrayOutputStream baos = new ByteArrayOutputStream();
     final PrintStream stdOut = new PrintStream(baos);
     try {
-      System.setOut(stdOut);
+      //System.setOut(stdOut);
       try {
         NameNode.createNameNode(new String[] { "-metadataVersion" }, conf);
       } catch (Exception e) {
@@ -89,7 +89,7 @@ public class TestMetadataVersionOutput {
       assertTrue(baos.toString("UTF-8").
           contains("Software format version: " + verNumStr));
     } finally {
-      System.setOut(origOut);
+      //System.setOut(origOut);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index 8bd7967..719cd89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -1052,8 +1052,8 @@ public class TestSnapshotDeletion {
     PrintStream oldOut = System.out;
     PrintStream oldErr = System.err;
     try {
-      System.setOut(psOut);
-      System.setErr(psOut);
+      //System.setOut(psOut);
+      //System.setErr(psOut);
       FsShell shell = new FsShell();
       shell.setConf(conf);
 
@@ -1072,8 +1072,8 @@ public class TestSnapshotDeletion {
       psOut.close();
       out.close();
     } finally {
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 90e499d..3dc8df4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -212,8 +212,8 @@ public class TestSnapshotFileLength {
     PrintStream outBackup = System.out;
     PrintStream errBackup = System.err;
     ByteArrayOutputStream bao = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(bao));
-    System.setErr(new PrintStream(bao));
+    //System.setOut(new PrintStream(bao));
+    //System.setErr(new PrintStream(bao));
     // Make sure we can cat the file upto to snapshot length
     FsShell shell = new FsShell();
     try {
@@ -221,8 +221,8 @@ public class TestSnapshotFileLength {
       "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
       assertEquals("Unexpected # bytes from -cat", BLOCKSIZE, bao.size());
     } finally {
-      System.setOut(outBackup);
-      System.setErr(errBackup);
+      //System.setOut(outBackup);
+      //System.setErr(errBackup);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
index 2905fcf..a2ef169 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
@@ -257,8 +257,8 @@ public class TestSnapshotRename {
     PrintStream oldOut = System.out;
     PrintStream oldErr = System.err;
     try {
-      System.setOut(psOut);
-      System.setErr(psOut);
+      //System.setOut(psOut);
+      //System.setErr(psOut);
       FsShell shell = new FsShell();
       shell.setConf(conf);
 
@@ -277,8 +277,8 @@ public class TestSnapshotRename {
       psOut.close();
       out.close();
     } finally {
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index 58ad52c..d9ed683 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -105,8 +105,8 @@ public class TestDFSAdmin {
   }
 
   private void redirectStream() {
-    System.setOut(new PrintStream(out));
-    System.setErr(new PrintStream(err));
+    //System.setOut(new PrintStream(out));
+    //System.setErr(new PrintStream(err));
   }
 
   private void resetStream() {
@@ -120,8 +120,8 @@ public class TestDFSAdmin {
       System.out.flush();
       System.err.flush();
     } finally {
-      System.setOut(OLD_OUT);
-      System.setErr(OLD_ERR);
+      //System.setOut(OLD_OUT);
+      //System.setErr(OLD_ERR);
     }
 
     if (cluster != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
index 0874f0e..40260dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java
@@ -93,8 +93,8 @@ public class TestDFSAdminWithHA {
     admin.setConf(conf);
     assertTrue(HAUtil.isHAEnabled(conf, "ns1"));
 
-    System.setOut(new PrintStream(out));
-    System.setErr(new PrintStream(err));
+    //System.setOut(new PrintStream(out));
+    //System.setErr(new PrintStream(err));
 
     // Reduce the number of retries to speed up the tests.
     conf.setInt(
@@ -114,8 +114,8 @@ public class TestDFSAdminWithHA {
       System.out.flush();
       System.err.flush();
     } finally {
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
     if (admin != null) {
       admin.close();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
index 5c890a5..eb51580 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java
@@ -76,14 +76,14 @@ public class TestDebugAdmin {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldErr = System.err;
     final PrintStream oldOut = System.out;
-    System.setErr(out);
-    System.setOut(out);
+    //System.setErr(out);
+    //System.setOut(out);
     int ret;
     try {
       ret = admin.run(cmd);
     } finally {
-      System.setErr(oldErr);
-      System.setOut(oldOut);
+      //System.setErr(oldErr);
+      //System.setOut(oldOut);
       IOUtils.closeStream(out);
     }
     return "ret: " + ret + ", " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
index bbe1c2e..c064733 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
@@ -296,7 +296,7 @@ public class TestOfflineEditsViewer {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     try {
-      System.setOut(out);
+      //System.setOut(out);
       int status = new OfflineEditsViewer().run(new String[] { "-h" });
       assertTrue("" + "Exit code returned for help option is incorrect",
           status == 0);
@@ -304,7 +304,7 @@ public class TestOfflineEditsViewer {
           "Invalid Command error displayed when help option is passed.", bytes
               .toString().contains("Error parsing command-line options"));
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       IOUtils.closeStream(out);
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index d0a7567..6e4f989 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -428,7 +428,7 @@ public class TestOfflineImageViewer {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     try {
-      System.setOut(out);
+      //System.setOut(out);
       int status = OfflineImageViewerPB.run(new String[] { "-h" });
       assertTrue("Exit code returned for help option is incorrect", status == 0);
       Assert.assertFalse(
@@ -442,7 +442,7 @@ public class TestOfflineImageViewer {
           "Exit code returned for help with other option is incorrect",
           status == -1);
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       IOUtils.closeStream(out);
     }
   }
@@ -637,7 +637,7 @@ public class TestOfflineImageViewer {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     try {
-      System.setOut(out);
+      //System.setOut(out);
       // Add the -h option to make the test only for option parsing,
       // and don't need to do the following operations.
       OfflineImageViewer.main(new String[] {"-i", "-", "-o", "-", "-p",
@@ -645,7 +645,7 @@ public class TestOfflineImageViewer {
       Assert.assertFalse(bytes.toString().contains(
           "Error parsing command-line options: "));
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       IOUtils.closeStream(out);
     }
   }
@@ -656,7 +656,7 @@ public class TestOfflineImageViewer {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     try {
-      System.setOut(out);
+      //System.setOut(out);
       int status =
           OfflineImageViewerPB.run(new String[] {"-i",
               originalFsimage.getAbsolutePath(), "-o", "-", "-p",
@@ -665,7 +665,7 @@ public class TestOfflineImageViewer {
       assertEquals(0, status);
       Assert.assertTrue(bytes.toString().contains("(0 B, 8 B]"));
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       IOUtils.closeStream(out);
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
index a61420c..9c123be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestJMXGet.java
@@ -143,7 +143,7 @@ public class TestJMXGet {
     PipedOutputStream pipeOut = new PipedOutputStream();
     PipedInputStream pipeIn = new PipedInputStream(pipeOut, 1024 * 1024);
     PrintStream oldErr = System.err;
-    System.setErr(new PrintStream(pipeOut));
+    //System.setErr(new PrintStream(pipeOut));
     try {
       jmx.printAllValues();
       if ((size = pipeIn.available()) != 0) {
@@ -153,7 +153,7 @@ public class TestJMXGet {
       pipeOut.close();
       pipeIn.close();
     } finally {
-      System.setErr(oldErr);
+      //System.setErr(oldErr);
     }
     return bytes != null ? new String(bytes).contains(pattern) : false;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
index e2f1011..ef6e6c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestTools.java
@@ -110,9 +110,9 @@ public class TestTools {
       PipedOutputStream pipeOut = new PipedOutputStream();
       PipedInputStream pipeIn = new PipedInputStream(pipeOut, PIPE_BUFFER_SIZE);
       if (out == System.out) {
-        System.setOut(new PrintStream(pipeOut));
+        //System.setOut(new PrintStream(pipeOut));
       } else if (out == System.err) {
-        System.setErr(new PrintStream(pipeOut));
+        //System.setErr(new PrintStream(pipeOut));
       }
 
       if (clazz == DelegationTokenFetcher.class) {
@@ -129,8 +129,8 @@ public class TestTools {
     } catch (Exception ex) {
       fail("checkOutput error " + ex);
     } finally {
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
index ee9494b..fdd49ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tracing/TestTraceAdmin.java
@@ -51,8 +51,8 @@ public class TestTraceAdmin extends SaslDataTransferTestCase {
     PrintStream ps = new PrintStream(baos);
     PrintStream oldStdout = System.out;
     PrintStream oldStderr = System.err;
-    System.setOut(ps);
-    System.setErr(ps);
+    //System.setOut(ps);
+    //System.setErr(ps);
     int ret = -1;
     try {
       ret = trace.run(cmd);
@@ -60,8 +60,8 @@ public class TestTraceAdmin extends SaslDataTransferTestCase {
       try {
         System.out.flush();
       } finally {
-        System.setOut(oldStdout);
-        System.setErr(oldStderr);
+        //System.setOut(oldStdout);
+        //System.setErr(oldStderr);
       }
     }
     return "ret:" + ret + ", " + baos.toString();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
index cb5301f..2d6bc14 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
@@ -350,7 +350,7 @@ public class TestJobHistoryParsing {
     // test output for HistoryViewer
     PrintStream stdps = System.out;
     try {
-      System.setOut(new PrintStream(outContent));
+      //System.setOut(new PrintStream(outContent));
       HistoryViewer viewer;
       synchronized (fileInfo) {
         viewer = new HistoryViewer(fc.makeQualified(
@@ -370,7 +370,7 @@ public class TestJobHistoryParsing {
             taskInfo.getTaskId().toString()) > 0);
       }
     } finally {
-      System.setOut(stdps);
+      //System.setOut(stdps);
 
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
index 45c7d1f..84d0021 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
@@ -325,13 +325,13 @@ public class TestNetworkedJob {
       JobQueueClient jobClient = new JobQueueClient(job);
 
       ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
+      //System.setOut(new PrintStream(bytes));
       String[] arg = { "-list" };
       jobClient.run(arg);
       assertTrue(bytes.toString().contains("Queue Name : default"));
       assertTrue(bytes.toString().contains("Queue State : running"));
       bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
+      //System.setOut(new PrintStream(bytes));
       String[] arg1 = { "-showacls" };
       jobClient.run(arg1);
       assertTrue(bytes.toString().contains("Queue acls for user :"));
@@ -343,7 +343,7 @@ public class TestNetworkedJob {
       // test for info and default queue
 
       bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
+      //System.setOut(new PrintStream(bytes));
       String[] arg2 = { "-info", "default" };
       jobClient.run(arg2);
       assertTrue(bytes.toString().contains("Queue Name : default"));
@@ -352,7 +352,7 @@ public class TestNetworkedJob {
 
       // test for info , default queue and jobs
       bytes = new ByteArrayOutputStream();
-      System.setOut(new PrintStream(bytes));
+      //System.setOut(new PrintStream(bytes));
       String[] arg3 = { "-info", "default", "-showJobs" };
       jobClient.run(arg3);
       assertTrue(bytes.toString().contains("Queue Name : default"));
@@ -365,7 +365,7 @@ public class TestNetworkedJob {
 
       
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
       if (fileSys != null) {
         fileSys.delete(testDir, true);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
index 9b88c5c..d103af0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java
@@ -272,7 +272,7 @@ public class TestPipeApplication {
     ExitUtil.disableSystemExit();
     // test without parameters
     try {
-      System.setOut(new PrintStream(out));
+      //System.setOut(new PrintStream(out));
       Submitter.main(new String[0]);
       fail();
     } catch (ExitUtil.ExitException e) {
@@ -321,7 +321,7 @@ public class TestPipeApplication {
           "-archives <archive1,...>          specify a comma-separated list of "
               + "archives to be unarchived on the compute machines"));
     } finally {
-      System.setOut(oldps);
+      //System.setOut(oldps);
       // restore
       System.setSecurityManager(securityManager);
       if (psw != null) {
@@ -372,7 +372,7 @@ public class TestPipeApplication {
       assertEquals(e.status, 0);
 
     } finally {
-      System.setOut(oldps);
+      //System.setOut(oldps);
       System.setSecurityManager(securityManager);
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 747cd9f..a9e6314 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -95,10 +95,10 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
     PrintStream oldOut = System.out;
     PrintStream newOut = new PrintStream(out, true);
     try {
-      System.setOut(newOut);
+      //System.setOut(newOut);
       return ToolRunner.run(conf, tool, args);
     } finally {
-      System.setOut(oldOut);
+      //System.setOut(oldOut);
     }
   }
 
@@ -265,7 +265,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
   private void startStop() {
     ByteArrayOutputStream data = new ByteArrayOutputStream();
     PrintStream error = System.err;
-    System.setErr(new PrintStream(data));
+    //System.setErr(new PrintStream(data));
     ExitUtil.disableSystemExit();
     try {
       CLI.main(new String[0]);
@@ -277,7 +277,7 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
     } catch (Exception e) {
 
     } finally {
-      System.setErr(error);
+      //System.setErr(error);
     }
     // in console should be written help text 
     String s = new String(data.toByteArray());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java
index 36119f5..1856d5c 100644
--- a/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java
+++ b/hadoop-tools/hadoop-ant/src/main/java/org/apache/hadoop/ant/DfsTask.java
@@ -148,8 +148,8 @@ public class DfsTask extends Task {
     antOut = System.out;
     antErr = System.err;
     try {
-      System.setOut(new PrintStream(out, false, "UTF-8"));
-      System.setErr(out == err ?
+      //System.setOut(new PrintStream(out, false, "UTF-8"));
+      //System.setErr(out == err ?
           System.out : new PrintStream(err, false, "UTF-8"));
     } catch (UnsupportedEncodingException ignored) {
     }
@@ -167,8 +167,8 @@ public class DfsTask extends Task {
     if (out != err && errprop != null && !System.err.checkError())
       getProject().setNewProperty(errprop, err.toString());
 
-    System.setErr(antErr);
-    System.setOut(antOut);
+    //System.setErr(antErr);
+    //System.setOut(antOut);
     confloader.cleanup();
     confloader.setParent(null);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
index c63bc42..02bb5c5 100644
--- a/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
+++ b/hadoop-tools/hadoop-archives/src/test/java/org/apache/hadoop/tools/TestHadoopArchives.java
@@ -186,7 +186,7 @@ public class TestHadoopArchives {
     PrintStream stderr = System.err;
     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
     PrintStream newErr = new PrintStream(byteStream);
-    System.setErr(newErr);
+    //System.setErr(newErr);
 
     // fail if the archive path already exists
     createFile(archivePath, fs, harName);
@@ -210,7 +210,7 @@ public class TestHadoopArchives {
     Assert.assertTrue(output.indexOf("Destination " + archivePath2.toString()
         + " should be a directory but is a file") != -1);
 
-    System.setErr(stderr);
+    //System.setErr(stderr);
   }
 
   @Test
@@ -296,16 +296,16 @@ public class TestHadoopArchives {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
-    System.setOut(out);
-    System.setErr(out);
+    //System.setOut(out);
+    //System.setErr(out);
     final String results;
     try {
       Assert.assertEquals(0, shell.run(new String[] { "-lsr", dir }));
       results = bytes.toString();
     } finally {
       IOUtils.closeStream(out);
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
     System.out.println("lsr results:\n" + results);
     String dirname = rootDir;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
index fea374e..d853c0b 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java
@@ -86,16 +86,16 @@ public abstract class TestDistCpSyncReverseBase {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
-    System.setOut(out);
-    System.setErr(out);
+    //System.setOut(out);
+    //System.setErr(out);
     final String results;
     try {
       Assert.assertEquals(0, shell.run(new String[] {"-lsr", dir }));
       results = bytes.toString();
     } finally {
       IOUtils.closeStream(out);
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
     System.out.println("lsr results:\n" + results);
     String dirname = rootDir;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
index b2266b3..5d7faf5 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
@@ -107,10 +107,10 @@ public class TestDistCpSystem {
     ByteArrayOutputStream baout = new ByteArrayOutputStream();
     PrintStream out = new PrintStream(baout, true);
     PrintStream old = System.out;
-    System.setOut(out);
+    //System.setOut(out);
     shell.run(args);
     out.close();
-    System.setOut(old);
+    //System.setOut(old);
     return baout.toString();
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
index b28da4c..a89bc6f 100644
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
+++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
@@ -1115,10 +1115,10 @@ public class TestCopyFiles extends TestCase {
     ByteArrayOutputStream baout = new ByteArrayOutputStream();
     PrintStream out = new PrintStream(baout, true);
     PrintStream old = System.out;
-    System.setOut(out);
+    //System.setOut(out);
     shell.run(args);
     out.close();
-    System.setOut(old);
+    //System.setOut(old);
     return baout.toString();
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
index 3704c5b..b4eb893 100644
--- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
+++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestDistCh.java
@@ -206,16 +206,16 @@ public class TestDistCh extends junit.framework.TestCase {
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
     final PrintStream oldErr = System.err;
-    System.setOut(out);
-    System.setErr(out);
+    //System.setOut(out);
+    //System.setErr(out);
     final String results;
     try {
       assertEquals(returnvalue, shell.run(new String[]{"-lsr", root}));
       results = bytes.toString();
     } finally {
       IOUtils.closeStream(out);
-      System.setOut(oldOut);
-      System.setErr(oldErr);
+      //System.setOut(oldOut);
+      //System.setErr(oldErr);
     }
     System.out.println("results:\n" + results);
     return results;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
index f849843..f3c86c4 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
@@ -179,7 +179,7 @@ public class TestGridmixSubmission extends CommonJobTest {
     final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bytes);
     final PrintStream oldOut = System.out;
-    System.setErr(out);
+    //System.setErr(out);
     ExitUtil.disableSystemExit();
     try {
       String[] argv = new String[0];
@@ -189,7 +189,7 @@ public class TestGridmixSubmission extends CommonJobTest {
       assertExceptionContains(ExitUtil.EXIT_EXCEPTION_MESSAGE, e);
       ExitUtil.resetFirstExitException();
     } finally {
-      System.setErr(oldOut);
+      //System.setErr(oldOut);
       System.setSecurityManager(securityManager);
     }
     String print = bytes.toString();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java
index bbfc020..479a7b3 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java
@@ -45,7 +45,7 @@ public class TestDumpTypedBytes {
     PrintStream psBackup = System.out;
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     PrintStream psOut = new PrintStream(out);
-    System.setOut(psOut);
+    //System.setOut(psOut);
     DumpTypedBytes dumptb = new DumpTypedBytes(conf);
 
     try {
@@ -86,7 +86,7 @@ public class TestDumpTypedBytes {
         fs.close();
       } catch (Exception e) {
       }
-      System.setOut(psBackup);
+      //System.setOut(psBackup);
       cluster.shutdown();
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
index 31e4905..640602f 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UtilTest.java
@@ -80,8 +80,8 @@ class UtilTest {
       new File(antTestDir_).mkdirs();
       File outFile = new File(antTestDir_, testName_+".log");
       PrintStream out = new PrintStream(new FileOutputStream(outFile));
-      System.setOut(out);
-      System.setErr(out);
+      //System.setOut(out);
+      //System.setErr(out);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
index 5a0f049..6b959e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
@@ -50,7 +50,7 @@ public class TestClusterCLI {
     sysOut = spy(new PrintStream(sysOutStream));
     sysErrStream = new ByteArrayOutputStream();
     sysErr = spy(new PrintStream(sysErrStream));
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
   }
   
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 7aa92f3..f539e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -99,11 +99,11 @@ public class TestLogsCLI {
   public void setUp() {
     sysOutStream = new ByteArrayOutputStream();
     sysOut =  new PrintStream(sysOutStream);
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
 
     sysErrStream = new ByteArrayOutputStream();
     sysErr = new PrintStream(sysErrStream);
-    System.setErr(sysErr);
+    //System.setErr(sysErr);
   }
 
   @Test(timeout = 5000l)


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[47/50] [abbrv] hadoop git commit: HADOOP-15919. AliyunOSS: Enable Yarn to use OSS. Contributed by wujinhu.

Posted by jh...@apache.org.
HADOOP-15919. AliyunOSS: Enable Yarn to use OSS. Contributed by wujinhu.

(cherry picked from commit be0708c6eb3c759e43639e0f8df7c9910308fb97)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6de44c06
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6de44c06
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6de44c06

Branch: refs/heads/branch-2-jhung-test
Commit: 6de44c060a4b4afb346fe73414bb3e826992b124
Parents: 762df75
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Nov 19 14:00:54 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 hadoop-tools/hadoop-aliyun/pom.xml              |   6 +
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      |   7 +-
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java |   2 -
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |   2 +
 .../org/apache/hadoop/fs/aliyun/oss/OSS.java    |  43 +++++
 .../site/markdown/tools/hadoop-aliyun/index.md  |  10 ++
 .../fs/aliyun/oss/AliyunOSSTestUtils.java       |  19 ++-
 .../oss/TestAliyunOSSBlockOutputStream.java     |   3 +
 .../aliyun/oss/contract/AliyunOSSContract.java  |   9 ++
 .../oss/fileContext/TestOSSFileContext.java     |  28 ++++
 .../TestOSSFileContextCreateMkdir.java          |  40 +++++
 .../TestOSSFileContextMainOperations.java       |  71 ++++++++
 .../TestOSSFileContextStatistics.java           |  69 ++++++++
 .../oss/fileContext/TestOSSFileContextURI.java  |  51 ++++++
 .../oss/fileContext/TestOSSFileContextUtil.java |  40 +++++
 .../hadoop/fs/aliyun/oss/yarn/TestOSS.java      |  83 ++++++++++
 .../aliyun/oss/yarn/TestOSSMiniYarnCluster.java | 160 +++++++++++++++++++
 17 files changed, 635 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
index 8968952..ea53868 100644
--- a/hadoop-tools/hadoop-aliyun/pom.xml
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -143,5 +143,11 @@
       <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-examples</artifactId>
+      <scope>test</scope>
+      <type>jar</type>
+    </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 809c8c8..cfd4cc0 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -136,7 +136,7 @@ public class AliyunOSSFileSystem extends FileSystem {
             key,
             uploadPartSize,
             new SemaphoredDelegatingExecutor(boundedThreadPool,
-                blockOutputActiveBlocks, true)), (Statistics)(null));
+                blockOutputActiveBlocks, true)), statistics);
   }
 
   /**
@@ -298,6 +298,11 @@ public class AliyunOSSFileSystem extends FileSystem {
   }
 
   @Override
+  public int getDefaultPort() {
+    return Constants.OSS_DEFAULT_PORT;
+  }
+
+  @Override
   public Path getWorkingDirectory() {
     return workingDir;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index f0413e3..bdd677d 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -416,7 +416,6 @@ public class AliyunOSSFileSystemStore {
       PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
       LOG.debug(result.getETag());
       statistics.incrementWriteOps(1);
-      statistics.incrementBytesWritten(file.length());
     } finally {
       fis.close();
     }
@@ -618,7 +617,6 @@ public class AliyunOSSFileSystemStore {
         uploadRequest.setPartNumber(idx);
         UploadPartResult uploadResult = ossClient.uploadPart(uploadRequest);
         statistics.incrementWriteOps(1);
-        statistics.incrementBytesWritten(file.length());
         return uploadResult.getPartETag();
       } catch (Exception e) {
         LOG.debug("Failed to upload "+ file.getPath() +", " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 0e0f64c..fdc7235 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -38,6 +38,8 @@ public final class Constants {
   public static final String CREDENTIALS_PROVIDER_KEY =
       "fs.oss.credentials.provider";
 
+  public static final int OSS_DEFAULT_PORT = -1;
+
   // OSS access verification
   public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId";
   public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
new file mode 100644
index 0000000..d545138
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSS.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * OSS implementation of AbstractFileSystem.
+ * This impl delegates to the AliyunOSSFileSystem.
+ */
+public class OSS extends DelegateToFileSystem {
+
+  public OSS(URI theUri, Configuration conf)
+          throws IOException, URISyntaxException {
+    super(theUri, new AliyunOSSFileSystem(), conf, "oss", false);
+  }
+
+  @Override
+  public int getUriDefaultPort() {
+    return Constants.OSS_DEFAULT_PORT;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 87aa90b..425fee5 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -111,6 +111,16 @@ please raise your issues with them.
 ### Other properties
 
     <property>
+      <name>fs.AbstractFileSystem.oss.impl</name>
+      <value>org.apache.hadoop.fs.aliyun.oss.OSS</value>
+      <description>The implementation class of the OSS AbstractFileSystem.
+        If you want to use OSS as YARN’s resource storage dir via the
+        fs.defaultFS configuration property in Hadoop’s core-site.xml,
+        you should add this configuration to Hadoop's core-site.xml
+      </description>
+    </property>
+
+    <property>
       <name>fs.oss.endpoint</name>
       <description>Aliyun OSS endpoint to connect to. An up-to-date list is
         provided in the Aliyun OSS Documentation.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
index 901cb2b..5248dea 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.aliyun.oss;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
 import org.junit.internal.AssumptionViolatedException;
 
 import java.io.IOException;
@@ -45,10 +46,21 @@ public final class AliyunOSSTestUtils {
    */
   public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
       throws IOException {
+    AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
+    ossfs.initialize(getURI(conf), conf);
+    return ossfs;
+  }
+
+  public static FileContext createTestFileContext(Configuration conf) throws
+      IOException {
+    return FileContext.getFileContext(getURI(conf), conf);
+  }
+
+  private static URI getURI(Configuration conf) {
     String fsname = conf.getTrimmed(
         TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");
 
-    boolean liveTest = StringUtils.isNotEmpty(fsname);
+    boolean liveTest = !StringUtils.isEmpty(fsname);
     URI testURI = null;
     if (liveTest) {
       testURI = URI.create(fsname);
@@ -59,11 +71,8 @@ public final class AliyunOSSTestUtils {
       throw new AssumptionViolatedException("No test filesystem in "
           + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
     }
-    AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
-    ossfs.initialize(testURI, conf);
-    return ossfs;
+    return testURI;
   }
-
   /**
    * Generate unique test path for multiple user tests.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
index c3387a3..47a2494 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
@@ -75,6 +75,7 @@ public class TestAliyunOSSBlockOutputStream {
 
   @Test
   public void testRegularUpload() throws IOException {
+    FileSystem.clearStatistics();
     long size = 1024 * 1024;
     FileSystem.Statistics statistics =
         FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
@@ -111,6 +112,7 @@ public class TestAliyunOSSBlockOutputStream {
   @Test
   public void testMultiPartUpload() throws IOException {
     long size = 6 * 1024 * 1024;
+    FileSystem.clearStatistics();
     FileSystem.Statistics statistics =
         FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
     ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size - 1);
@@ -134,6 +136,7 @@ public class TestAliyunOSSBlockOutputStream {
 
   @Test
   public void testMultiPartUploadConcurrent() throws IOException {
+    FileSystem.clearStatistics();
     long size = 50 * 1024 * 1024 - 1;
     ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size);
     FileSystem.Statistics statistics =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
index 624c606..fd0afca 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
@@ -19,9 +19,13 @@
 package org.apache.hadoop.fs.aliyun.oss.contract;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
 
+import java.io.IOException;
+
 /**
  * The contract of Aliyun OSS: only enabled if the test bucket is provided.
  */
@@ -41,6 +45,11 @@ public class AliyunOSSContract extends AbstractBondedFSContract {
   }
 
   @Override
+  public FileSystem getTestFileSystem() throws IOException {
+    return AliyunOSSTestUtils.createTestFileSystem(new Configuration());
+  }
+
+  @Override
   public Path getTestPath() {
     String testUniqueForkId = System.getProperty("test.unique.fork.id");
     return testUniqueForkId == null ? super.getTestPath() :

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
new file mode 100644
index 0000000..530797b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContext.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.fs.TestFileContext;
+
+/**
+ * Implementation of TestFileContext for OSS.
+ */
+public class TestOSSFileContext extends TestFileContext {
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
new file mode 100644
index 0000000..7bddbd0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextCreateMkdir.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextCreateMkdirBaseTest.
+ */
+public class TestOSSFileContextCreateMkdir
+    extends FileContextCreateMkdirBaseTest {
+
+  @Before
+  public void setUp() throws IOException, Exception {
+    Configuration conf = new Configuration();
+    fc = AliyunOSSTestUtils.createTestFileContext(conf);
+    super.setUp();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
new file mode 100644
index 0000000..36d02df
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextMainOperations.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextMainOperationsBaseTest;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextMainOperationsBaseTest.
+ */
+public class TestOSSFileContextMainOperations
+    extends FileContextMainOperationsBaseTest {
+
+  @Before
+  public void setUp() throws IOException, Exception {
+    Configuration conf = new Configuration();
+    fc = AliyunOSSTestUtils.createTestFileContext(conf);
+    super.setUp();
+  }
+
+  @Override
+  protected boolean listCorruptedBlocksSupported() {
+    return false;
+  }
+
+  @Test
+  @Ignore
+  public void testCreateFlagAppendExistingFile() throws IOException {
+    // append not supported, so test removed
+  }
+
+  @Test
+  @Ignore
+  public void testCreateFlagCreateAppendExistingFile() throws IOException {
+    // append not supported, so test removed
+  }
+
+  @Test
+  @Ignore
+  public void testSetVerifyChecksum() throws IOException {
+    // checksums ignored, so test ignored
+  }
+
+  @Test
+  @Ignore
+  public void testBuilderCreateAppendExistingFile() throws IOException {
+    // append not supported, so test removed
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
new file mode 100644
index 0000000..cbb7d85
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextStatistics.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FCStatisticsBaseTest;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+import java.net.URI;
+
+/**
+ * OSS implementation of FCStatisticsBaseTest.
+ */
+public class TestOSSFileContextStatistics extends FCStatisticsBaseTest {
+
+  @Before
+  public void setUp() throws Exception {
+    Configuration conf = new Configuration();
+    fc = AliyunOSSTestUtils.createTestFileContext(conf);
+    fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"),
+        FileContext.DEFAULT_PERM, true);
+    FileContext.clearStatistics();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (fc != null) {
+      fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true);
+    }
+  }
+
+  @Override
+  protected void verifyReadBytes(FileSystem.Statistics stats) {
+    // one blockSize for read, one for pread
+    Assert.assertEquals(2 * blockSize, stats.getBytesRead());
+  }
+
+  @Override
+  protected void verifyWrittenBytes(FileSystem.Statistics stats) {
+    // no extra bytes are written
+    Assert.assertEquals(blockSize, stats.getBytesWritten());
+  }
+
+  @Override
+  protected URI getFsUri() {
+    return fc.getHomeDirectory().toUri();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
new file mode 100644
index 0000000..cde708e
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextURI.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextURIBase;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextURIBase.
+ */
+public class TestOSSFileContextURI extends FileContextURIBase {
+
+  @Before
+  public void setUp() throws IOException, Exception {
+    Configuration conf = new Configuration();
+    fc1 = AliyunOSSTestUtils.createTestFileContext(conf);
+    // different object, same FS
+    fc2 = AliyunOSSTestUtils.createTestFileContext(conf);
+    super.setUp();
+  }
+
+  @Test
+  @Ignore
+  public void testFileStatus() throws IOException {
+    // test ignored
+    // (the statistics tested with this method are not relevant for an OSSFS)
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
new file mode 100644
index 0000000..ce55c54
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/fileContext/TestOSSFileContextUtil.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.fs.aliyun.oss.fileContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContextUtilBase;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+
+/**
+ * OSS implementation of FileContextUtilBase.
+ */
+public class TestOSSFileContextUtil extends FileContextUtilBase {
+
+  @Before
+  public void setUp() throws IOException, Exception {
+    Configuration conf = new Configuration();
+    fc = AliyunOSSTestUtils.createTestFileContext(conf);
+    super.setUp();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
new file mode 100644
index 0000000..08ef60f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSS.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FsStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.EnumSet;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * OSS tests through the {@link FileContext} API.
+ */
+public class TestOSS {
+  private FileContext fc;
+
+  @Before
+  public void setUp() throws Exception {
+    Configuration conf = new Configuration();
+    fc = AliyunOSSTestUtils.createTestFileContext(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    if (fc != null) {
+      fc.delete(getTestPath(), true);
+    }
+  }
+
+  protected Path getTestPath() {
+    return new Path(AliyunOSSTestUtils.generateUniqueTestPath());
+  }
+
+  @Test
+  public void testOSSStatus() throws Exception {
+    FsStatus fsStatus = fc.getFsStatus(null);
+    assertNotNull(fsStatus);
+    assertTrue("Used capacity should be positive: " + fsStatus.getUsed(),
+        fsStatus.getUsed() >= 0);
+    assertTrue("Remaining capacity should be positive: " + fsStatus
+        .getRemaining(),
+        fsStatus.getRemaining() >= 0);
+    assertTrue("Capacity should be positive: " + fsStatus.getCapacity(),
+        fsStatus.getCapacity() >= 0);
+  }
+
+  @Test(timeout = 90000L)
+  public void testOSSCreateFileInSubDir() throws Exception {
+    Path dirPath = getTestPath();
+    fc.mkdir(dirPath, FileContext.DIR_DEFAULT_PERM, true);
+    Path filePath = new Path(dirPath, "file");
+    try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag
+        .CREATE))) {
+      file.write(666);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6de44c06/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
new file mode 100644
index 0000000..439aa0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/yarn/TestOSSMiniYarnCluster.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.examples.WordCount;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.aliyun.oss.AliyunOSSTestUtils;
+import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests that OSS is usable through a YARN application.
+ */
+public class TestOSSMiniYarnCluster extends AbstractFSContractTestBase {
+
+  private Configuration conf;
+  private FileSystem fs;
+  private MiniYARNCluster yarnCluster;
+  private Path rootPath;
+
+  @Override
+  protected AbstractFSContract createContract(Configuration configuration) {
+    this.conf = configuration;
+    return new AliyunOSSContract(conf);
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    fs = getFileSystem();
+    rootPath = path("MiniClusterWordCount");
+    Path workingDir = path("working");
+    fs.setWorkingDirectory(workingDir);
+    fs.mkdirs(new Path(rootPath, "input/"));
+
+    yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName
+            1, // number of node managers
+            1, // number of local log dirs per node manager
+            1); // number of hdfs dirs per node manager
+    yarnCluster.init(conf);
+    yarnCluster.start();
+  }
+
+  @Test
+  public void testWithMiniCluster() throws Exception {
+    Path input = new Path(rootPath, "input/in");
+    input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+    Path output = new Path(rootPath, "output/");
+    output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory());
+
+    writeStringToFile(input, "first line\nsecond line\nthird line");
+
+    Job job = Job.getInstance(conf, "word count");
+    job.setJarByClass(WordCount.class);
+    job.setMapperClass(WordCount.TokenizerMapper.class);
+    job.setCombinerClass(WordCount.IntSumReducer.class);
+    job.setReducerClass(WordCount.IntSumReducer.class);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(IntWritable.class);
+    FileInputFormat.addInputPath(job, input);
+    FileOutputFormat.setOutputPath(job, output);
+
+    int exitCode = (job.waitForCompletion(true) ? 0 : 1);
+    assertEquals("Returned error code.", 0, exitCode);
+
+    assertTrue(fs.exists(new Path(output, "_SUCCESS")));
+    String outputAsStr = readStringFromFile(new Path(output, "part-r-00000"));
+    Map<String, Integer> resAsMap = getResultAsMap(outputAsStr);
+
+    assertEquals(4, resAsMap.size());
+    assertEquals(1, (int) resAsMap.get("first"));
+    assertEquals(1, (int) resAsMap.get("second"));
+    assertEquals(1, (int) resAsMap.get("third"));
+    assertEquals(3, (int) resAsMap.get("line"));
+  }
+
+
+  /**
+   * helper method.
+   */
+  private Map<String, Integer> getResultAsMap(String outputAsStr) {
+    Map<String, Integer> result = new HashMap<>();
+    for (String line : outputAsStr.split("\n")) {
+      String[] tokens = line.split("\t");
+      assertTrue("Not enough tokens in in string \" "
+              + line + "\" from output \"" + outputAsStr + "\"",
+          tokens.length > 1);
+      result.put(tokens[0], Integer.parseInt(tokens[1]));
+    }
+    return result;
+  }
+
+  /**
+   * helper method.
+   */
+  private void writeStringToFile(Path path, String string) throws IOException {
+    FileContext fc = AliyunOSSTestUtils.createTestFileContext(conf);
+    try (FSDataOutputStream file = fc.create(path,
+        EnumSet.of(CreateFlag.CREATE))) {
+      file.write(string.getBytes());
+    }
+  }
+
+  /**
+   * helper method.
+   */
+  private String readStringFromFile(Path path) throws IOException {
+    try (FSDataInputStream in = fs.open(path)) {
+      long bytesLen = fs.getFileStatus(path).getLen();
+      byte[] buffer = new byte[(int) bytesLen];
+      IOUtils.readFully(in, buffer, 0, buffer.length);
+      return new String(buffer);
+    }
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    if (yarnCluster != null) {
+      yarnCluster.stop();
+    }
+    super.teardown();
+    IOUtils.closeStream(getFileSystem());
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[25/50] [abbrv] hadoop git commit: MAPREDUCE-7156. NullPointerException when reaching max shuffle connections. Contributed by Peter Bacsko

Posted by jh...@apache.org.
MAPREDUCE-7156. NullPointerException when reaching max shuffle connections. Contributed by Peter Bacsko

(cherry picked from commit ba1f9d66d94ed0b85084d7c40c09a87478b3a05a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b86c6512
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b86c6512
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b86c6512

Branch: refs/heads/branch-2-jhung-test
Commit: b86c651202fa1fa15e29919ba34264b510073928
Parents: ba89181
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Nov 6 17:55:51 2018 -0600
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b86c6512/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 0bab750..51cfa54 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -911,6 +911,8 @@ public class ShuffleHandler extends AuxiliaryService {
     @Override
     public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt) 
         throws Exception {
+      super.channelOpen(ctx, evt);
+
       if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) {
         LOG.info(String.format("Current number of shuffle connections (%d) is " + 
             "greater than or equal to the max allowed shuffle connections (%d)", 
@@ -926,8 +928,6 @@ public class ShuffleHandler extends AuxiliaryService {
         return;
       }
       accepted.add(evt.getChannel());
-      super.channelOpen(ctx, evt);
-     
     }
 
     @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[27/50] [abbrv] hadoop git commit: Update jdiff.stable.api from 2.9.1 to 2.9.2.

Posted by jh...@apache.org.
Update jdiff.stable.api from 2.9.1 to 2.9.2.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a213ea22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a213ea22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a213ea22

Branch: refs/heads/branch-2-jhung-test
Commit: a213ea2236c8e5b045f92ac3632e6a61d7e896fb
Parents: 1d539c5
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 20 14:32:49 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 hadoop-project-dist/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a213ea22/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 1519651..a038cd3 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -144,7 +144,7 @@
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
-        <jdiff.stable.api>2.9.1</jdiff.stable.api>
+        <jdiff.stable.api>2.9.2</jdiff.stable.api>
         <jdiff.stability>-unstable</jdiff.stability>
         <!-- Commented out for HADOOP-11776 -->
         <!-- Uncomment param name="${jdiff.compatibility}" in javadoc doclet if compatibility is not empty -->


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[44/50] [abbrv] hadoop git commit: HADOOP-15925. The config and log of gpg-agent are removed in create-release script. Contributed by Dinesh Chitlangia.

Posted by jh...@apache.org.
HADOOP-15925. The config and log of gpg-agent are removed in create-release script. Contributed by Dinesh Chitlangia.

(cherry picked from commit af5958f7b6a209d6a6622acee47a66a4a921828d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc3e9792
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc3e9792
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc3e9792

Branch: refs/heads/branch-2-jhung-test
Commit: fc3e97926df872819dd6b217260da1d0d17b6ef9
Parents: 914716d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Nov 16 13:16:59 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 dev-support/bin/create-release | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc3e9792/dev-support/bin/create-release
----------------------------------------------------------------------
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index 33c87cd..a089e7c 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -504,7 +504,7 @@ function makearelease
   big_console_header "Cleaning the Source Tree"
 
   # git clean to clear any remnants from previous build
-  run "${GIT}" clean -xdf
+  run "${GIT}" clean -xdf -e /patchprocess
 
   mkdir -p "${LOGDIR}"
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[46/50] [abbrv] hadoop git commit: HADOOP-15943. AliyunOSS: add missing owner & group attributes for oss FileStatus. Contributed by wujinhu.

Posted by jh...@apache.org.
HADOOP-15943. AliyunOSS: add missing owner & group attributes for oss FileStatus. Contributed by wujinhu.

(cherry picked from commit 5ff0cf86a940fd83f1425794921cc075b19f1108)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e320762a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e320762a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e320762a

Branch: refs/heads/branch-2-jhung-test
Commit: e320762a2948ead1876aa56d4c420eaefec978f5
Parents: a213ea2
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Nov 23 13:56:54 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      | 25 +++++++------
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 11 +++---
 .../hadoop/fs/aliyun/oss/OSSFileStatus.java     | 38 ++++++++++++++++++++
 .../oss/TestAliyunOSSFileSystemContract.java    | 13 +++++++
 4 files changed, 72 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e320762a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index cfd4cc0..531652b 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
 import org.apache.hadoop.util.Progressable;
 
@@ -70,6 +71,7 @@ public class AliyunOSSFileSystem extends FileSystem {
       LoggerFactory.getLogger(AliyunOSSFileSystem.class);
   private URI uri;
   private String bucket;
+  private String username;
   private Path workingDir;
   private int blockOutputActiveBlocks;
   private AliyunOSSFileSystemStore store;
@@ -259,7 +261,7 @@ public class AliyunOSSFileSystem extends FileSystem {
 
     // Root always exists
     if (key.length() == 0) {
-      return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+      return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username);
     }
 
     ObjectMetadata meta = store.getObjectMetadata(key);
@@ -273,17 +275,17 @@ public class AliyunOSSFileSystem extends FileSystem {
       ObjectListing listing = store.listObjects(key, 1, null, false);
       if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) ||
           CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) {
-        return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+        return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username);
       } else {
         throw new FileNotFoundException(path + ": No such file or directory!");
       }
     } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
-      return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
-          qualifiedPath);
+      return new OSSFileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
+          qualifiedPath, username);
     } else {
-      return new FileStatus(meta.getContentLength(), false, 1,
+      return new OSSFileStatus(meta.getContentLength(), false, 1,
           getDefaultBlockSize(path), meta.getLastModified().getTime(),
-          qualifiedPath);
+          qualifiedPath, username);
     }
   }
 
@@ -330,15 +332,16 @@ public class AliyunOSSFileSystem extends FileSystem {
 
     bucket = name.getHost();
     uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
-    workingDir = new Path("/user",
-        System.getProperty("user.name")).makeQualified(uri, null);
+    // Username is the current user at the time the FS was instantiated.
+    username = UserGroupInformation.getCurrentUser().getShortUserName();
+    workingDir = new Path("/user", username).makeQualified(uri, null);
     long keepAliveTime = longOption(conf,
         KEEPALIVE_TIME_KEY, KEEPALIVE_TIME_DEFAULT, 0);
     blockOutputActiveBlocks = intOption(conf,
         UPLOAD_ACTIVE_BLOCKS_KEY, UPLOAD_ACTIVE_BLOCKS_DEFAULT, 1);
 
     store = new AliyunOSSFileSystemStore();
-    store.initialize(name, conf, statistics);
+    store.initialize(name, conf, username, statistics);
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
 
     int threadNum = AliyunOSSUtils.intPositiveOption(conf,
@@ -475,9 +478,9 @@ public class AliyunOSSFileSystem extends FileSystem {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Adding: fi: " + keyPath);
             }
-            result.add(new FileStatus(objectSummary.getSize(), false, 1,
+            result.add(new OSSFileStatus(objectSummary.getSize(), false, 1,
                 getDefaultBlockSize(keyPath),
-                objectSummary.getLastModified().getTime(), keyPath));
+                objectSummary.getLastModified().getTime(), keyPath, username));
           }
         }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e320762a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index bdd677d..855644e 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -82,6 +82,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 public class AliyunOSSFileSystemStore {
   public static final Logger LOG =
       LoggerFactory.getLogger(AliyunOSSFileSystemStore.class);
+  private String username;
   private FileSystem.Statistics statistics;
   private OSSClient ossClient;
   private String bucketName;
@@ -90,8 +91,9 @@ public class AliyunOSSFileSystemStore {
   private int maxKeys;
   private String serverSideEncryptionAlgorithm;
 
-  public void initialize(URI uri, Configuration conf,
+  public void initialize(URI uri, Configuration conf, String user,
                          FileSystem.Statistics stat) throws IOException {
+    this.username = user;
     statistics = stat;
     ClientConfiguration clientConf = new ClientConfiguration();
     clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
@@ -573,9 +575,9 @@ public class AliyunOSSFileSystemStore {
           String key = summary.getKey();
           Path path = fs.makeQualified(new Path("/" + key));
           if (filter.accept(path) && acceptor.accept(path, summary)) {
-            FileStatus status = new FileStatus(summary.getSize(),
+            FileStatus status = new OSSFileStatus(summary.getSize(),
                 key.endsWith("/"), 1, fs.getDefaultBlockSize(path),
-                summary.getLastModified().getTime(), path);
+                summary.getLastModified().getTime(), path, username);
             stats.add(status);
           }
         }
@@ -583,7 +585,8 @@ public class AliyunOSSFileSystemStore {
         for(String commonPrefix: listing.getCommonPrefixes()) {
           Path path = fs.makeQualified(new Path("/" + commonPrefix));
           if (filter.accept(path) && acceptor.accept(path, commonPrefix)) {
-            FileStatus status = new FileStatus(0, true, 1, 0, 0, path);
+            FileStatus status = new OSSFileStatus(0, true, 1, 0, 0,
+                path, username);
             stats.add(status);
           }
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e320762a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
new file mode 100644
index 0000000..13ed8d8
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * This class is used by listStatus for oss files.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class OSSFileStatus extends FileStatus {
+  public OSSFileStatus(long length, boolean isdir, int blockReplication,
+      long blocksize, long modTime, Path path, String user) {
+    super(length, isdir, blockReplication, blocksize, modTime, path);
+    setOwner(user);
+    setGroup(user);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e320762a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
index 0570146..81759a5 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystemContractBaseTest;
 import org.apache.hadoop.fs.Path;
 
+import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -83,6 +84,18 @@ public class TestAliyunOSSFileSystemContract
   }
 
   @Test
+  public void testListStatus() throws IOException {
+    Path file = this.path("/test/hadoop/file");
+    this.createFile(file);
+    assertTrue("File exists", this.fs.exists(file));
+    FileStatus fs = this.fs.getFileStatus(file);
+    assertEquals(fs.getOwner(),
+        UserGroupInformation.getCurrentUser().getShortUserName());
+    assertEquals(fs.getGroup(),
+        UserGroupInformation.getCurrentUser().getShortUserName());
+  }
+
+  @Test
   public void testDeleteSubdir() throws IOException {
     Path parentDir = this.path("/test/hadoop");
     Path file = this.path("/test/hadoop/file");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[35/50] [abbrv] hadoop git commit: YARN-9085. Add Guaranteed and MaxCapacity to CSQueueMetrics

Posted by jh...@apache.org.
YARN-9085. Add Guaranteed and MaxCapacity to CSQueueMetrics

(cherry picked from commit cc51607ccdf60f4ebecf7afa3e2056297ca0ac5f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6669c8bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6669c8bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6669c8bf

Branch: refs/heads/branch-2-jhung-test
Commit: 6669c8bf9ae3beb37d8bd730e3c0dd042a964ee8
Parents: 46f0f4c
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Dec 7 10:32:53 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/CSQueueMetrics.java      | 39 +++++++++++++++++
 .../scheduler/capacity/CSQueueUtils.java        | 17 ++++++++
 .../scheduler/capacity/LeafQueue.java           |  4 ++
 .../scheduler/capacity/ParentQueue.java         |  4 ++
 .../capacity/TestCapacityScheduler.java         | 44 ++++++++++++++++++++
 5 files changed, 108 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6669c8bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 87fc234..8382665 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.metrics2.annotation.Metric;
 import org.apache.hadoop.metrics2.annotation.Metrics;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.metrics2.lib.MutableGaugeFloat;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
 import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -46,6 +47,14 @@ public class CSQueueMetrics extends QueueMetrics {
   MutableGaugeFloat usedCapacity;
   @Metric("Percent of Absolute Capacity Used")
   MutableGaugeFloat absoluteUsedCapacity;
+  @Metric("Guaranteed memory in MB")
+  MutableGaugeLong guaranteedMB;
+  @Metric("Guaranteed CPU in virtual cores")
+  MutableGaugeInt guaranteedVCores;
+  @Metric("Maximum memory in MB")
+  MutableGaugeLong maxCapacityMB;
+  @Metric("Maximum CPU in virtual cores")
+  MutableGaugeInt maxCapacityVCores;
 
   CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
       boolean enableUserMetrics, Configuration conf) {
@@ -126,6 +135,36 @@ public class CSQueueMetrics extends QueueMetrics {
     }
   }
 
+  public long getGuaranteedMB() {
+    return guaranteedMB.value();
+  }
+
+  public int getGuaranteedVCores() {
+    return guaranteedVCores.value();
+  }
+
+  public void setGuaranteedResources(String partition, Resource res) {
+    if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      guaranteedMB.set(res.getMemorySize());
+      guaranteedVCores.set(res.getVirtualCores());
+    }
+  }
+
+  public long getMaxCapacityMB() {
+    return maxCapacityMB.value();
+  }
+
+  public int getMaxCapacityVCores() {
+    return maxCapacityVCores.value();
+  }
+
+  public void setMaxCapacityResources(String partition, Resource res) {
+    if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      maxCapacityMB.set(res.getMemorySize());
+      maxCapacityVCores.set(res.getVirtualCores());
+    }
+  }
+
   public synchronized static CSQueueMetrics forQueue(String queueName,
       Queue parent, boolean enableUserMetrics, Configuration conf) {
     MetricsSystem ms = DefaultMetricsSystem.instance();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6669c8bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 9554086..da1d304 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -295,4 +295,21 @@ class CSQueueUtils {
     childQueue.getMetrics().setAvailableResourcesToQueue(nodePartition,
         getMaxAvailableResourceToQueue(rc, nlm, childQueue, cluster));
    }
+
+  /**
+   * Updated configured capacity/max-capacity for queue.
+   * @param rc resource calculator
+   * @param partitionResource total cluster resources for this partition
+   * @param partition partition being updated
+   * @param queue queue
+   */
+   public static void updateConfiguredCapacityMetrics(ResourceCalculator rc,
+       Resource partitionResource, String partition, AbstractCSQueue queue) {
+     queue.getMetrics().setGuaranteedResources(partition, rc.multiplyAndNormalizeDown(
+         partitionResource, queue.getQueueCapacities().getAbsoluteCapacity(partition),
+         queue.getMinimumAllocation()));
+     queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown(
+         partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition),
+         queue.getMinimumAllocation()));
+   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6669c8bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 41c6f09..c352ba4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1794,6 +1794,10 @@ public class LeafQueue extends AbstractCSQueue {
       // Update metrics
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
           this, labelManager, null);
+      // Update configured capacity/max-capacity for default partition only
+      CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator,
+          labelManager.getResourceByLabel(null, clusterResource),
+          RMNodeLabelsManager.NO_LABEL, this);
 
       // queue metrics are updated, more resource may be available
       // activate the pending applications if possible

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6669c8bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 213bf07..2de13cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -841,6 +841,10 @@ public class ParentQueue extends AbstractCSQueue {
 
       CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
           this, labelManager, null);
+      // Update configured capacity/max-capacity for default partition only
+      CSQueueUtils.updateConfiguredCapacityMetrics(resourceCalculator,
+          labelManager.getResourceByLabel(null, clusterResource),
+          RMNodeLabelsManager.NO_LABEL, this);
     } finally {
       writeLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6669c8bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 5713c43..d23ef59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -5209,4 +5209,48 @@ public class TestCapacityScheduler {
     assertEquals(4, appsInB1.size());
     rm.close();
   }
+
+  @Test
+  public void testCSQueueMetrics() throws Exception {
+    CapacityScheduler cs = new CapacityScheduler();
+    cs.setConf(new YarnConfiguration());
+    cs.setRMContext(resourceManager.getRMContext());
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+    setupQueueConfiguration(conf);
+    cs.init(conf);
+    cs.start();
+
+    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 1, "n1");
+    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 2, "n2");
+    cs.handle(new NodeAddedSchedulerEvent(n1));
+    cs.handle(new NodeAddedSchedulerEvent(n2));
+
+    assertEquals(10240, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB());
+    assertEquals(71680, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
+    assertEquals(102400, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
+    assertEquals(102400, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+
+    // Remove a node, metrics should be updated
+    cs.handle(new NodeRemovedSchedulerEvent(n2));
+    assertEquals(5120, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB());
+    assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
+    assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
+    assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+
+    // Add child queue to a, and reinitialize. Metrics should be updated
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );
+    conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 30.0f);
+    conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.0f);
+    conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 50.0f);
+
+    cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+        null, new RMContainerTokenSecretManager(conf),
+        new NMTokenSecretManagerInRM(conf),
+        new ClientToAMTokenSecretManagerInRM(), null));
+
+    assertEquals(1024, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getGuaranteedMB());
+    assertEquals(2048, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getGuaranteedMB());
+    assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getMaxCapacityMB());
+    assertEquals(25600, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getMaxCapacityMB());
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[38/50] [abbrv] hadoop git commit: HADOOP-15923. create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent.

Posted by jh...@apache.org.
HADOOP-15923. create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent.

(cherry picked from commit 703b2860a49577629e7b3ef461d8a61292e79c88)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f64b4516
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f64b4516
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f64b4516

Branch: refs/heads/branch-2-jhung-test
Commit: f64b4516c9d06402060db456ea4fe4866b984fb8
Parents: 00b4455
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 13 13:40:43 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 dev-support/bin/create-release | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f64b4516/dev-support/bin/create-release
----------------------------------------------------------------------
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index 39c9133..fbe3fb2 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -259,7 +259,8 @@ function startgpgagent
   if [[ "${SIGN}" = true ]]; then
     if [[ -n "${GPGAGENT}" && -z "${GPG_AGENT_INFO}" ]]; then
       echo "starting gpg agent"
-      echo "default-cache-ttl 14400" > "${LOGDIR}/gpgagent.conf"
+      echo "default-cache-ttl 36000" > "${LOGDIR}/gpgagent.conf"
+      echo "max-cache-ttl 36000" >> "${LOGDIR}/gpgagent.conf"
       # shellcheck disable=2046
       eval $("${GPGAGENT}" --daemon \
         --options "${LOGDIR}/gpgagent.conf" \


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[32/50] [abbrv] hadoop git commit: Revert "HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak."

Posted by jh...@apache.org.
Revert "HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak."

This reverts commit b98ffbe3f228b706b8e547f82e134233ab691443.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1219559
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1219559
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1219559

Branch: refs/heads/branch-2-jhung-test
Commit: a1219559d1f8a660989ae06c04153a1a016cbb61
Parents: 1f8c879
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Dec 3 13:22:14 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop/security/LdapGroupsMapping.java      | 163 ++++++-------------
 .../src/main/resources/core-default.xml         |  28 +---
 .../src/site/markdown/GroupsMapping.md          |  54 +-----
 .../hadoop/security/TestLdapGroupsMapping.java  |  80 +++++----
 .../security/TestLdapGroupsMappingBase.java     |  76 +--------
 .../TestLdapGroupsMappingWithFailover.java      | 142 ----------------
 .../TestLdapGroupsMappingWithOneQuery.java      |  16 +-
 .../TestLdapGroupsMappingWithPosixGroup.java    |  10 +-
 8 files changed, 119 insertions(+), 450 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 83eb5ad..6beaa9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -25,7 +25,6 @@ import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Hashtable;
-import java.util.Iterator;
 import java.util.List;
 import java.util.HashSet;
 import java.util.Collection;
@@ -41,10 +40,7 @@ import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 import javax.naming.ldap.LdapName;
 import javax.naming.ldap.Rdn;
-import javax.naming.spi.InitialContextFactory;
 
-import com.google.common.collect.Iterators;
-import com.sun.jndi.ldap.LdapCtxFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
@@ -87,7 +83,7 @@ public class LdapGroupsMapping
   public static final String LDAP_CONFIG_PREFIX = "hadoop.security.group.mapping.ldap";
 
   /*
-   * URL of the LDAP server(s)
+   * URL of the LDAP server
    */
   public static final String LDAP_URL_KEY = LDAP_CONFIG_PREFIX + ".url";
   public static final String LDAP_URL_DEFAULT = "";
@@ -236,20 +232,6 @@ public class LdapGroupsMapping
       LDAP_CONFIG_PREFIX + ".read.timeout.ms";
   public static final int READ_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds
 
-  public static final String LDAP_NUM_ATTEMPTS_KEY =
-      LDAP_CONFIG_PREFIX + ".num.attempts";
-  public static final int LDAP_NUM_ATTEMPTS_DEFAULT = 3;
-
-  public static final String LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY =
-      LDAP_CONFIG_PREFIX + ".num.attempts.before.failover";
-  public static final int LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT =
-      LDAP_NUM_ATTEMPTS_DEFAULT;
-
-  public static final String LDAP_CTX_FACTORY_CLASS_KEY =
-      LDAP_CONFIG_PREFIX + ".ctx.factory.class";
-  public static final Class<? extends LdapCtxFactory>
-      LDAP_CTX_FACTORY_CLASS_DEFAULT = LdapCtxFactory.class;
-
   private static final Logger LOG =
       LoggerFactory.getLogger(LdapGroupsMapping.class);
 
@@ -260,10 +242,8 @@ public class LdapGroupsMapping
 
   private DirContext ctx;
   private Configuration conf;
-
-  private Iterator<String> ldapUrls;
-  private String currentLdapUrl;
-
+  
+  private String ldapUrl;
   private boolean useSsl;
   private String keystore;
   private String keystorePass;
@@ -278,15 +258,14 @@ public class LdapGroupsMapping
   private String memberOfAttr;
   private String groupMemberAttr;
   private String groupNameAttr;
-  private int groupHierarchyLevels;
+  private int    groupHierarchyLevels;
   private String posixUidAttr;
   private String posixGidAttr;
   private boolean isPosix;
   private boolean useOneQuery;
-  private int numAttempts;
-  private int numAttemptsBeforeFailover;
-  private Class<? extends InitialContextFactory> ldapCxtFactoryClass;
 
+  public static final int RECONNECT_RETRY_COUNT = 3;
+  
   /**
    * Returns list of groups for a user.
    * 
@@ -300,31 +279,20 @@ public class LdapGroupsMapping
   @Override
   public synchronized List<String> getGroups(String user) {
     /*
-     * Normal garbage collection takes care of removing Context instances when
-     * they are no longer in use. Connections used by Context instances being
-     * garbage collected will be closed automatically. So in case connection is
-     * closed and gets CommunicationException, retry some times with new new
-     * DirContext/connection.
+     * Normal garbage collection takes care of removing Context instances when they are no longer in use. 
+     * Connections used by Context instances being garbage collected will be closed automatically.
+     * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. 
      */
-
-    // Tracks the number of attempts made using the same LDAP server
-    int atemptsBeforeFailover = 1;
-
-    for (int attempt = 1; attempt <= numAttempts; attempt++,
-        atemptsBeforeFailover++) {
+    for(int retry = 0; retry < RECONNECT_RETRY_COUNT; retry++) {
       try {
         return doGetGroups(user, groupHierarchyLevels);
       } catch (NamingException e) {
-        LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
-            "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
+        LOG.warn("Failed to get groups for user " + user + " (retry=" + retry
+            + ") by " + e);
         LOG.trace("TRACE", e);
-
-        if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
-          atemptsBeforeFailover = 0;
-        }
       }
 
-      // Reset ctx so that new DirContext can be created with new connection
+      //reset ctx so that new DirContext can be created with new connection
       this.ctx = null;
     }
     
@@ -410,10 +378,10 @@ public class LdapGroupsMapping
   private List<String> lookupGroup(SearchResult result, DirContext c,
       int goUpHierarchy)
       throws NamingException {
-    List<String> groups = new ArrayList<>();
-    Set<String> groupDNs = new HashSet<>();
+    List<String> groups = new ArrayList<String>();
+    Set<String> groupDNs = new HashSet<String>();
 
-    NamingEnumeration<SearchResult> groupResults;
+    NamingEnumeration<SearchResult> groupResults = null;
     // perform the second LDAP query
     if (isPosix) {
       groupResults = lookupPosixGroup(result, c);
@@ -434,10 +402,10 @@ public class LdapGroupsMapping
       }
       if (goUpHierarchy > 0 && !isPosix) {
         // convert groups to a set to ensure uniqueness
-        Set<String> groupset = new HashSet<>(groups);
+        Set<String> groupset = new HashSet<String>(groups);
         goUpGroupHierarchy(groupDNs, goUpHierarchy, groupset);
         // convert set back to list for compatibility
-        groups = new ArrayList<>(groupset);
+        groups = new ArrayList<String>(groupset);
       }
     }
     return groups;
@@ -465,9 +433,11 @@ public class LdapGroupsMapping
         userSearchFilter, new Object[]{user}, SEARCH_CONTROLS);
     // return empty list if the user can not be found.
     if (!results.hasMoreElements()) {
-      LOG.debug("doGetGroups({}) returned no groups because the " +
-          "user is not found.", user);
-      return new ArrayList<>();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("doGetGroups(" + user + ") returned no groups because the " +
+            "user is not found.");
+      }
+      return new ArrayList<String>();
     }
     SearchResult result = results.nextElement();
 
@@ -485,7 +455,7 @@ public class LdapGroupsMapping
               memberOfAttr + "' attribute." +
               "Returned user object: " + result.toString());
         }
-        groups = new ArrayList<>();
+        groups = new ArrayList<String>();
         NamingEnumeration groupEnumeration = groupDNAttr.getAll();
         while (groupEnumeration.hasMore()) {
           String groupDN = groupEnumeration.next().toString();
@@ -500,7 +470,9 @@ public class LdapGroupsMapping
     if (groups == null || groups.isEmpty() || goUpHierarchy > 0) {
       groups = lookupGroup(result, c, goUpHierarchy);
     }
-    LOG.debug("doGetGroups({}) returned {}", user, groups);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("doGetGroups(" + user + ") returned " + groups);
+    }
     return groups;
   }
 
@@ -508,7 +480,7 @@ public class LdapGroupsMapping
   */
   void getGroupNames(SearchResult groupResult, Collection<String> groups,
                      Collection<String> groupDNs, boolean doGetDNs)
-                     throws NamingException {
+                     throws NamingException  {
     Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
     if (groupName == null) {
       throw new NamingException("The group object does not have " +
@@ -545,7 +517,7 @@ public class LdapGroupsMapping
       return;
     }
     DirContext context = getDirContext();
-    Set<String> nextLevelGroups = new HashSet<>();
+    Set<String> nextLevelGroups = new HashSet<String>();
     StringBuilder filter = new StringBuilder();
     filter.append("(&").append(groupSearchFilter).append("(|");
     for (String dn : groupDNs) {
@@ -565,32 +537,13 @@ public class LdapGroupsMapping
     goUpGroupHierarchy(nextLevelGroups, goUpHierarchy - 1, groups);
   }
 
-  /**
-   * Check whether we should fail over to the next LDAP server.
-   * @param attemptsMadeWithSameLdap current number of attempts made
-   *                                 with using same LDAP instance
-   * @param maxAttemptsBeforeFailover maximum number of attempts
-   *                                  before failing over
-   * @return true if we should fail over to the next LDAP server
-   */
-  protected boolean failover(
-      int attemptsMadeWithSameLdap, int maxAttemptsBeforeFailover) {
-    if (attemptsMadeWithSameLdap >= maxAttemptsBeforeFailover) {
-      String previousLdapUrl = currentLdapUrl;
-      currentLdapUrl = ldapUrls.next();
-      LOG.info("Reached {} attempts on {}, failing over to {}",
-          attemptsMadeWithSameLdap, previousLdapUrl, currentLdapUrl);
-      return true;
-    }
-    return false;
-  }
-
-  private DirContext getDirContext() throws NamingException {
+  DirContext getDirContext() throws NamingException {
     if (ctx == null) {
       // Set up the initial environment for LDAP connectivity
-      Hashtable<String, String> env = new Hashtable<>();
-      env.put(Context.INITIAL_CONTEXT_FACTORY, ldapCxtFactoryClass.getName());
-      env.put(Context.PROVIDER_URL, currentLdapUrl);
+      Hashtable<String, String> env = new Hashtable<String, String>();
+      env.put(Context.INITIAL_CONTEXT_FACTORY,
+          com.sun.jndi.ldap.LdapCtxFactory.class.getName());
+      env.put(Context.PROVIDER_URL, ldapUrl);
       env.put(Context.SECURITY_AUTHENTICATION, "simple");
 
       // Set up SSL security, if necessary
@@ -628,7 +581,7 @@ public class LdapGroupsMapping
    * Caches groups, no need to do that for this provider
    */
   @Override
-  public void cacheGroupsRefresh() {
+  public void cacheGroupsRefresh() throws IOException {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -638,7 +591,7 @@ public class LdapGroupsMapping
    * @param groups unused
    */
   @Override
-  public void cacheGroupsAdd(List<String> groups) {
+  public void cacheGroupsAdd(List<String> groups) throws IOException {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -649,12 +602,10 @@ public class LdapGroupsMapping
 
   @Override
   public synchronized void setConf(Configuration conf) {
-    String[] urls = conf.getStrings(LDAP_URL_KEY, LDAP_URL_DEFAULT);
-    if (urls == null || urls.length == 0) {
-      throw new RuntimeException("LDAP URL(s) are not configured");
+    ldapUrl = conf.get(LDAP_URL_KEY, LDAP_URL_DEFAULT);
+    if (ldapUrl == null || ldapUrl.isEmpty()) {
+      throw new RuntimeException("LDAP URL is not configured");
     }
-    ldapUrls = Iterators.cycle(urls);
-    currentLdapUrl = ldapUrls.next();
 
     useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
     if (useSsl) {
@@ -670,13 +621,17 @@ public class LdapGroupsMapping
     
     String baseDN = conf.getTrimmed(BASE_DN_KEY, BASE_DN_DEFAULT);
 
-    // User search base which defaults to base dn.
+    //User search base which defaults to base dn.
     userbaseDN = conf.getTrimmed(USER_BASE_DN_KEY, baseDN);
-    LOG.debug("Usersearch baseDN: {}", userbaseDN);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Usersearch baseDN: " + userbaseDN);
+    }
 
-    // Group search base which defaults to base dn.
+    //Group search base which defaults to base dn.
     groupbaseDN = conf.getTrimmed(GROUP_BASE_DN_KEY, baseDN);
-    LOG.debug("Groupsearch baseDN: {}", groupbaseDN);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Groupsearch baseDN: " + userbaseDN);
+    }
 
     groupSearchFilter =
         conf.get(GROUP_SEARCH_FILTER_KEY, GROUP_SEARCH_FILTER_DEFAULT);
@@ -700,8 +655,7 @@ public class LdapGroupsMapping
     posixGidAttr =
         conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
 
-    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
-        DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
+    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
     SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
     // Limit the attributes returned to only those required to speed up the search.
     // See HADOOP-10626 and HADOOP-12001 for more details.
@@ -715,26 +669,9 @@ public class LdapGroupsMapping
     }
     SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
 
-    ldapCxtFactoryClass = conf.getClass(LDAP_CTX_FACTORY_CLASS_KEY,
-        LDAP_CTX_FACTORY_CLASS_DEFAULT, InitialContextFactory.class);
-
-    this.numAttempts = conf.getInt(LDAP_NUM_ATTEMPTS_KEY,
-        LDAP_NUM_ATTEMPTS_DEFAULT);
-    this.numAttemptsBeforeFailover = conf.getInt(
-        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
-        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT);
-
     this.conf = conf;
   }
 
-  /**
-   * Get URLs of configured LDAP servers.
-   * @return URLs of LDAP servers being used.
-   */
-  public Iterator<String> getLdapUrls() {
-    return ldapUrls;
-  }
-
   private void loadSslConf(Configuration sslConf) {
     keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
     keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY,
@@ -784,8 +721,8 @@ public class LdapGroupsMapping
         password = new String(passchars);
       }
     } catch (IOException ioe) {
-      LOG.warn("Exception while trying to get password for alias {}:",
-          alias, ioe);
+      LOG.warn("Exception while trying to get password for alias " + alias
+              + ": ", ioe);
     }
     return password;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e5ce728..e262b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -234,35 +234,11 @@
 </property>
 
 <property>
-  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
-  <value>3</value>
-  <description>
-    This property is the number of attempts to be made for LDAP operations.
-    If this limit is exceeded, LdapGroupsMapping will return an empty
-    group list.
-  </description>
-</property>
-
-<property>
-  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
-  <value>3</value>
-  <description>
-    This property is the number of attempts to be made for LDAP operations
-    using a single LDAP instance. If multiple LDAP servers are configured
-    and this number of failed operations is reached, we will switch to the
-    next LDAP server. The configuration for the overall number of attempts
-    will still be respected, failover will thus be performed only if this
-    property is less than hadoop.security.group.mapping.ldap.num.attempts.
-  </description>
-</property>
-
-<property>
   <name>hadoop.security.group.mapping.ldap.url</name>
   <value></value>
   <description>
-    The URL of the LDAP server(s) to use for resolving user groups when using
-    the LdapGroupsMapping user to group mapping. Supports configuring multiple
-    LDAP servers via a comma-separated list.
+    The URL of the LDAP server to use for resolving user groups when using
+    the LdapGroupsMapping user to group mapping.
   </description>
 </property>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index b957821..c6af930 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -73,7 +73,7 @@ The cache invalidation is configurable via `hadoop.security.groups.negative-cach
 LDAP Groups Mapping
 --------
 This provider supports LDAP with simple password authentication using JNDI API.
-`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server(s) for resolving user groups. It supports configuring multiple LDAP servers via a comma-separated list.
+`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server for resolving user groups.
 
 `hadoop.security.group.mapping.ldap.base` configures the search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.
 Get groups for a given username first looks up the user and then looks up the groups for the user result. If the directory setup has different user and group search bases, use `hadoop.security.group.mapping.ldap.userbase` and `hadoop.security.group.mapping.ldap.groupbase` configs.
@@ -115,58 +115,6 @@ To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memb
 If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`.
 Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`.
 
-### Configuring retries and multiple LDAP servers with failover ###
-If there are issues encountered when retrieving information from LDAP servers, the request will be retried. To configure the number of retries, use the following configuration:
-
-```<property>
-     <name>hadoop.security.group.mapping.ldap.num.attempts</name>
-     <value>3</value>
-     <description>
-       This property is the number of attempts to be made for LDAP operations.
-       If this limit is exceeded, LdapGroupsMapping will return an empty
-       group list.
-     </description>
-    </property>
-```
-
-LDAP Groups Mapping also supports configuring multiple LDAP servers and failover if a particular instance is not available or is misbehaving.
-The following configuration shows configuring 3 LDAP servers. Additionally, 2 attempts will be made for each server before failing over to the next one, with 6 attempts overall before failing.
-
-```
-<property>
-  <name>hadoop.security.group.mapping.ldap.url</name>
-  <value>ldap://server1,ldap://server2,ldap://server3</value>
-  <description>
-    The URL of the LDAP server(s) to use for resolving user groups when using
-    the LdapGroupsMapping user to group mapping. Supports configuring multiple
-    LDAP servers via a comma-separated list.
-  </description>
-</property>
-
-<property>
-  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
-  <value>6</value>
-  <description>
-    This property is the number of attempts to be made for LDAP operations.
-    If this limit is exceeded, LdapGroupsMapping will return an empty
-    group list.
-  </description>
-</property>
-
-<property>
-  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
-  <value>2</value>
-  <description>
-    This property is the number of attempts to be made for LDAP operations
-    using a single LDAP instance. If multiple LDAP servers are configured
-    and this number of failed operations is reached, we will switch to the
-    next LDAP server. The configuration for the overall number of attempts
-    will still be respected, failover will thus be performed only if this
-    property is less than hadoop.security.group.mapping.ldap.num.attempts.
-  </description>
-</property>
-```
-
 Composite Groups Mapping
 --------
 `CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index a361d04..d9f0c2a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -18,12 +18,11 @@
 package org.apache.hadoop.security;
 
 import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT;
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
 import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
@@ -83,16 +82,14 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   private final String userDN = "CN=some_user,DC=test,DC=com";
 
-  private static final String TEST_LDAP_URL = "ldap://test";
-
   @Before
-  public void setupMocks() {
+  public void setupMocks() throws NamingException {
     when(getUserSearchResult().getNameInNamespace()).
         thenReturn(userDN);
   }
   
   @Test
-  public void testGetGroups() throws NamingException {
+  public void testGetGroups() throws IOException, NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -103,7 +100,9 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDifferentBaseDNs() throws Exception {
-    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
     String userBaseDN = "ou=Users,dc=xxx,dc=com ";
     String groupBaseDN = " ou=Groups,dc=xxx,dc=com";
     conf.set(LdapGroupsMapping.USER_BASE_DN_KEY, userBaseDN);
@@ -114,7 +113,9 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDefaultBaseDN() throws Exception {
-    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
     String baseDN = " dc=xxx,dc=com ";
     conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN);
     doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim());
@@ -158,7 +159,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithHierarchy() throws NamingException {
+  public void testGetGroupsWithHierarchy() throws IOException, NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     // The parent search is run once for each level, and is a different search
@@ -174,7 +175,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithConnectionClosed() throws NamingException {
+  public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
     // The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
     // then after reconnected return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -189,20 +190,21 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithLdapDown() throws NamingException {
+  public void testGetGroupsWithLdapDown() throws IOException, NamingException {
     // This mocks the case where Ldap server is down, and always throws CommunicationException 
     when(getContext().search(anyString(), anyString(), any(Object[].class),
         any(SearchControls.class)))
         .thenThrow(new CommunicationException("Connection is closed"));
     
     // Ldap server is down, no groups should be retrieved
-    doTestGetGroups(Arrays.asList(new String[] {}), 4);
+    doTestGetGroups(Arrays.asList(new String[] {}), 
+        LdapGroupsMapping.RECONNECT_RETRY_COUNT);
   }
   
-  private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
-      throws NamingException {
-    Configuration conf = getBaseConf(TEST_LDAP_URL);
-    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, searchTimes);
+  private void doTestGetGroups(List<String> expectedGroups, int searchTimes) throws IOException, NamingException {  
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(conf);
@@ -214,12 +216,17 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     
     // We should have searched for a user, and then two groups
     verify(getContext(), times(searchTimes)).search(anyString(),
-        anyString(), any(Object[].class), any(SearchControls.class));
+                                         anyString(),
+                                         any(Object[].class),
+                                         any(SearchControls.class));
   }
 
   private void doTestGetGroupsWithParent(List<String> expectedGroups,
-      int searchTimesGroup, int searchTimesParentGroup) throws NamingException {
-    Configuration conf = getBaseConf(TEST_LDAP_URL);
+      int searchTimesGroup, int searchTimesParentGroup)
+          throws IOException, NamingException {
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
     // Set the config to get parents 1 level up
     conf.setInt(LdapGroupsMapping.GROUP_HIERARCHY_LEVELS_KEY, 1);
 
@@ -230,14 +237,18 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     List<String> groups = groupsMapping.getGroups("some_user");
 
     // compare lists, ignoring the order
-    Assert.assertEquals(new HashSet<>(expectedGroups), new HashSet<>(groups));
+    Assert.assertEquals(new HashSet<String>(expectedGroups),
+        new HashSet<String>(groups));
 
     // We should have searched for a user, and group
     verify(getContext(), times(searchTimesGroup)).search(anyString(),
-        anyString(), any(Object[].class), any(SearchControls.class));
+                                         anyString(),
+                                         any(Object[].class),
+                                         any(SearchControls.class));
     // One groups search for the parent group should have been done
-    verify(getContext(), times(searchTimesParentGroup)).search(
-        anyString(), anyString(), any(SearchControls.class));
+    verify(getContext(), times(searchTimesParentGroup)).search(anyString(),
+                                         anyString(),
+                                         any(SearchControls.class));
   }
 
   @Test
@@ -257,7 +268,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   @Test
   public void testConfGetPassword() throws Exception {
     File testDir = GenericTestUtils.getTestDir();
-    Configuration conf = getBaseConf();
+    Configuration conf = new Configuration();
     final Path jksPath = new Path(testDir.toString(), "test.jks");
     final String ourUrl =
         JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
@@ -272,10 +283,10 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
 
     // ensure that we get nulls when the key isn't there
-    assertNull(provider.getCredentialEntry(
+    assertEquals(null, provider.getCredentialEntry(
         LdapGroupsMapping.BIND_PASSWORD_KEY));
-    assertNull(provider.getCredentialEntry(
-        LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
+    assertEquals(null, provider.getCredentialEntry
+        (LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
 
     // create new aliases
     try {
@@ -339,8 +350,9 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
-      final Configuration conf = getBaseConf(ldapUrl, null);
+      final Configuration conf = new Configuration();
+      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
+          "ldap://localhost:" + serverSock.getLocalPort());
       conf.setInt(CONNECTION_TIMEOUT, connectionTimeoutMs);
       mapping.setConf(conf);
 
@@ -394,8 +406,9 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
-      final Configuration conf = getBaseConf(ldapUrl, null);
+      final Configuration conf = new Configuration();
+      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
+          "ldap://localhost:" + serverSock.getLocalPort());
       conf.setInt(READ_TIMEOUT, readTimeoutMs);
       mapping.setConf(conf);
 
@@ -423,12 +436,15 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
    */
   @Test(timeout = 10000)
   public void testSetConf() throws Exception {
-    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    Configuration conf = new Configuration();
     Configuration mockConf = Mockito.spy(conf);
     when(mockConf.getPassword(anyString()))
         .thenThrow(new IOException("injected IOException"));
+    // Set a dummy LDAP server URL.
+    mockConf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(mockConf);
   }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
index 2469b4b..51d3673 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
@@ -18,15 +18,10 @@
 
 package org.apache.hadoop.security;
 
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_DEFAULT;
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_KEY;
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import javax.naming.Context;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
@@ -36,17 +31,13 @@ import javax.naming.directory.BasicAttributes;
 import javax.naming.directory.DirContext;
 import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
-import javax.naming.spi.InitialContextFactory;
 
-import org.apache.hadoop.conf.Configuration;
 import org.junit.Before;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
 
-import java.util.Hashtable;
-
 public class TestLdapGroupsMappingBase {
   @Mock
   private DirContext context;
@@ -71,6 +62,7 @@ public class TestLdapGroupsMappingBase {
   public void setupMocksBase() throws NamingException {
     MockitoAnnotations.initMocks(this);
     DirContext ctx = getContext();
+    doReturn(ctx).when(groupsMapping).getDirContext();
 
     when(ctx.search(Mockito.anyString(), Mockito.anyString(),
         Mockito.any(Object[].class), Mockito.any(SearchControls.class))).
@@ -125,30 +117,9 @@ public class TestLdapGroupsMappingBase {
         thenReturn("CN=some_group,DC=test,DC=com");
   }
 
-  protected Configuration getBaseConf() {
-    return getBaseConf("ldap://test");
-  }
-
-  protected Configuration getBaseConf(String ldapUrl) {
-    return getBaseConf(ldapUrl, getContext());
-  }
-
-  protected Configuration getBaseConf(
-      String ldapUrl, DirContext contextToReturn) {
-    DummyLdapCtxFactory.setContextToReturn(contextToReturn);
-    DummyLdapCtxFactory.setExpectedLdapUrl(ldapUrl);
-
-    Configuration conf = new Configuration();
-    conf.set(LDAP_URL_KEY, ldapUrl);
-    conf.setClass(LDAP_CTX_FACTORY_CLASS_KEY, DummyLdapCtxFactory.class,
-        InitialContextFactory.class);
-    return conf;
-  }
-
   protected DirContext getContext() {
     return context;
   }
-
   protected NamingEnumeration<SearchResult> getUserNames() {
     return userNames;
   }
@@ -168,56 +139,13 @@ public class TestLdapGroupsMappingBase {
   protected LdapGroupsMapping getGroupsMapping() {
     return groupsMapping;
   }
-
   protected String[] getTestGroups() {
     return testGroups;
   }
-
   protected NamingEnumeration getParentGroupNames() {
     return parentGroupNames;
   }
-
   protected String[] getTestParentGroups() {
     return testParentGroups;
   }
-
-  /**
-   * Ldap Context Factory implementation to be used for testing to check
-   * contexts are requested for the expected LDAP server URLs etc.
-   */
-  public static class DummyLdapCtxFactory implements InitialContextFactory {
-
-    private static DirContext contextToReturn;
-    private static String expectedLdapUrl;
-
-    public DummyLdapCtxFactory() {
-    }
-
-    protected static void setContextToReturn(DirContext ctx) {
-      contextToReturn = ctx;
-    }
-
-    protected static void setExpectedLdapUrl(String url) {
-      expectedLdapUrl = url;
-    }
-
-    @Override
-    public Context getInitialContext(Hashtable<?, ?> env)
-        throws NamingException {
-      if (expectedLdapUrl != null) {
-        String actualLdapUrl = (String) env.get(Context.PROVIDER_URL);
-        assertEquals(expectedLdapUrl, actualLdapUrl);
-      }
-      if (contextToReturn == null) {
-        InitialContextFactory defaultFactory = null;
-        try {
-          defaultFactory = LDAP_CTX_FACTORY_CLASS_DEFAULT.newInstance();
-        } catch (ReflectiveOperationException e) {
-          fail("Could not initialize the default factory");
-        }
-        return defaultFactory.getInitialContext(env);
-      }
-      return contextToReturn;
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
deleted file mode 100644
index 4f6ea1f..0000000
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import javax.naming.CommunicationException;
-import javax.naming.directory.SearchControls;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY;
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
-import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/**
- * Test failover functionality for switching to different
- * LDAP server URLs upon failures.
- */
-public class TestLdapGroupsMappingWithFailover
-    extends TestLdapGroupsMappingBase {
-
-  private static final String TEST_USER_NAME = "some_user";
-
-  /**
-   * Test that when disabled, we will retry the configured number
-   * of times using the same LDAP server.
-   */
-  @Test
-  public void testDoesNotFailoverWhenDisabled() throws Exception {
-    final int numAttempts = 3;
-    Configuration conf = getBaseConf();
-    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
-        "ldap://test2");
-    DummyLdapCtxFactory.setExpectedLdapUrl("ldap://test");
-    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
-    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY, numAttempts);
-
-    when(getContext().search(anyString(), anyString(), any(Object[].class),
-        any(SearchControls.class))).thenThrow(new CommunicationException());
-
-    LdapGroupsMapping groupsMapping = getGroupsMapping();
-    groupsMapping.setConf(conf);
-
-    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
-    assertTrue(groups.isEmpty());
-
-    // Test that we made 3 attempts using the same server
-    verify(getContext(), times(numAttempts)).search(anyString(), anyString(),
-        any(Object[].class), any(SearchControls.class));
-  }
-
-  /**
-   * Test that when configured, we will make the specified amount of
-   * attempts using one ldap url before failing over to the next one.
-   *
-   * This also tests that we wrap back to the first server
-   * if we've tried them all.
-   */
-  @Test
-  public void testFailover() throws Exception {
-    Queue<String> ldapUrls = new LinkedList<>();
-    ldapUrls.add("ldap://test");
-    ldapUrls.add("ldap://test1");
-    ldapUrls.add("ldap://test2");
-
-    final int numAttempts = 12;
-    final int numAttemptsBeforeFailover = 2;
-
-    Configuration conf = getBaseConf();
-    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
-        "ldap://test2");
-    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
-    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
-        numAttemptsBeforeFailover);
-
-    // Set the first expected url and add it back to the queue
-    String nextLdapUrl = ldapUrls.remove();
-    DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
-    ldapUrls.add(nextLdapUrl);
-
-    // Number of attempts using a single ldap server url
-    final AtomicInteger serverAttempts = new AtomicInteger(
-        numAttemptsBeforeFailover);
-
-    when(getContext().search(anyString(), anyString(), any(Object[].class),
-        any(SearchControls.class))).thenAnswer(new Answer<Object>() {
-          @Override
-          public Object answer(InvocationOnMock invocationOnMock)
-              throws Throwable {
-            if (serverAttempts.get() == 1) {
-              // Take the head of the queue and re-queue it to the back
-              String nextLdapUrl = ldapUrls.remove();
-              ldapUrls.add(nextLdapUrl);
-
-              DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
-              serverAttempts.set(numAttemptsBeforeFailover);
-            } else {
-              serverAttempts.decrementAndGet();
-            }
-            throw new CommunicationException();
-          }
-        });
-
-    LdapGroupsMapping groupsMapping = getGroupsMapping();
-    groupsMapping.setConf(conf);
-
-    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
-    assertTrue(groups.isEmpty());
-
-    // Test that we made 6 attempts overall
-    verify(getContext(), times(numAttempts)).search(anyString(),
-        anyString(), any(Object[].class), any(SearchControls.class));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
index f51a944..e5cd2b6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.security;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -67,7 +68,7 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   @Test
-  public void testGetGroups() throws NamingException {
+  public void testGetGroups() throws IOException, NamingException {
     // given a user whose ldap query returns a user object with three "memberOf"
     // properties, return an array of strings representing its groups.
     String[] testGroups = new String[] {"abc", "xyz", "sss"};
@@ -75,9 +76,10 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   private void doTestGetGroups(List<String> expectedGroups)
-      throws NamingException {
-    String ldapUrl = "ldap://test";
-    Configuration conf = getBaseConf(ldapUrl);
+      throws IOException, NamingException {
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
     // enable single-query lookup
     conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
 
@@ -90,7 +92,9 @@ public class TestLdapGroupsMappingWithOneQuery
     Assert.assertEquals(expectedGroups, groups);
 
     // We should have only made one query because single-query lookup is enabled
-    verify(getContext(), times(1)).search(anyString(), anyString(),
-        any(Object[].class), any(SearchControls.class));
+    verify(getContext(), times(1)).search(anyString(),
+        anyString(),
+        any(Object[].class),
+        any(SearchControls.class));
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1219559/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
index 4a21345..17d28a5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
@@ -28,6 +28,7 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -61,7 +62,7 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   @Test
-  public void testGetGroups() throws NamingException {
+  public void testGetGroups() throws IOException, NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), contains("posix"),
@@ -72,9 +73,10 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
-      throws NamingException {
-    String ldapUrl = "ldap://test";
-    Configuration conf = getBaseConf(ldapUrl);
+      throws IOException, NamingException {
+    Configuration conf = new Configuration();
+    // Set this, so we don't throw an exception
+    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
     conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY,
         "(objectClass=posixGroup)(cn={0})");
     conf.set(LdapGroupsMapping.USER_SEARCH_FILTER_KEY,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[12/50] [abbrv] hadoop git commit: HADOOP-15837. DynamoDB table Update can fail S3A FS init. Contributed by Steve Loughran.

Posted by jh...@apache.org.
HADOOP-15837. DynamoDB table Update can fail S3A FS init.
Contributed by Steve Loughran.

(cherry picked from commit f1fbc563719db88c50d57820b5052880cfd6b1aa)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec9a67f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec9a67f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec9a67f6

Branch: refs/heads/branch-2-jhung-test
Commit: ec9a67f644bee852f714dc8a68e9d0336765c567
Parents: be3f13d
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 12 09:49:28 2018 +0100
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec9a67f6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 458eb83..a5d0482 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -807,7 +807,6 @@ public class DynamoDBMetadataStore implements MetadataStore {
         final String status = description.getTableStatus();
         switch (status) {
         case "CREATING":
-        case "UPDATING":
           LOG.debug("Table {} in region {} is being created/updated. This may"
                   + " indicate that the table is being operated by another "
                   + "concurrent thread or process. Waiting for active...",
@@ -818,6 +817,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
           throw new FileNotFoundException("DynamoDB table "
               + "'" + tableName + "' is being "
               + "deleted in region " + region);
+        case "UPDATING":
+          // table being updated; it can still be used.
+          LOG.debug("Table is being updated.");
+          break;
         case "ACTIVE":
           break;
         default:


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[36/50] [abbrv] hadoop git commit: Add 2.9.2 release notes and changes documents.

Posted by jh...@apache.org.
Add 2.9.2 release notes and changes documents.

(cherry picked from commit 1a00b4e325146988375c9ce5b11016c45f059a4e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d539c55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d539c55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d539c55

Branch: refs/heads/branch-2-jhung-test
Commit: 1d539c551ae3f40b6f9457d49d0d8dcb5b3ed45e
Parents: 6de44c0
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 20 12:55:40 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../markdown/release/2.9.2/CHANGES.2.9.2.md     | 274 ++++++++++++++++
 .../release/2.9.2/RELEASENOTES.2.9.2.md         |  31 ++
 .../jdiff/Apache_Hadoop_HDFS_2.9.2.xml          | 312 +++++++++++++++++++
 3 files changed, 617 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d539c55/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGES.2.9.2.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGES.2.9.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGES.2.9.2.md
new file mode 100644
index 0000000..cf5e59b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/CHANGES.2.9.2.md
@@ -0,0 +1,274 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 2.9.2 - 2018-11-19
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode |  Major | datanode, hdfs | Lukas Majercak | Lukas Majercak |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-14987](https://issues.apache.org/jira/browse/HADOOP-14987) | Improve KMSClientProvider log around delegation token checking |  Major | . | Xiaoyu Yao | Xiaoyu Yao |
+| [YARN-7274](https://issues.apache.org/jira/browse/YARN-7274) | Ability to disable elasticity at leaf queue level |  Major | capacityscheduler | Scott Brokaw | Zian Chen |
+| [HADOOP-15394](https://issues.apache.org/jira/browse/HADOOP-15394) | Backport PowerShell NodeFencer HADOOP-14309 to branch-2 |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13462](https://issues.apache.org/jira/browse/HDFS-13462) | Add BIND\_HOST configuration for JournalNode's HTTP and RPC Servers |  Major | hdfs, journal-node | Lukas Majercak | Lukas Majercak |
+| [HADOOP-14841](https://issues.apache.org/jira/browse/HADOOP-14841) | Kms client should disconnect if unable to get output stream from connection. |  Major | kms | Xiao Chen | Rushabh S Shah |
+| [HDFS-13272](https://issues.apache.org/jira/browse/HDFS-13272) | DataNodeHttpServer to have configurable HttpServer2 threads |  Major | datanode | Erik Krogen | Erik Krogen |
+| [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. |  Minor | . | Wei-Chiu Chuang | Gabor Bota |
+| [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair |  Major | net | Nanda kumar | Nanda kumar |
+| [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover |  Critical | common | Karthik Palanisamy | Karthik Palanisamy |
+| [HDFS-13602](https://issues.apache.org/jira/browse/HDFS-13602) | Add checkOperation(WRITE) checks in FSNamesystem |  Major | ha, namenode | Erik Krogen | Chao Sun |
+| [HDFS-13653](https://issues.apache.org/jira/browse/HDFS-13653) | Make dfs.client.failover.random.order a per nameservice configuration |  Major | federation | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
+| [HDFS-13686](https://issues.apache.org/jira/browse/HDFS-13686) | Add overall metrics for FSNamesystemLock |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
+| [HDFS-13714](https://issues.apache.org/jira/browse/HDFS-13714) | Fix TestNameNodePrunesMissingStorages test failures on Windows |  Major | hdfs, namenode, test | Lukas Majercak | Lukas Majercak |
+| [HDFS-13719](https://issues.apache.org/jira/browse/HDFS-13719) | Docs around dfs.image.transfer.timeout are misleading |  Major | documentation | Kitti Nanasi | Kitti Nanasi |
+| [HDFS-11060](https://issues.apache.org/jira/browse/HDFS-11060) | make DEFAULT\_MAX\_CORRUPT\_FILEBLOCKS\_RETURNED configurable |  Minor | hdfs | Lantao Jin | Lantao Jin |
+| [HDFS-13813](https://issues.apache.org/jira/browse/HDFS-13813) | Exit NameNode if dangling child inode is detected when saving FsImage |  Major | hdfs, namenode | Siyao Meng | Siyao Meng |
+| [HDFS-13821](https://issues.apache.org/jira/browse/HDFS-13821) | RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache |  Major | hdfs | Fei Hui | Fei Hui |
+| [HADOOP-15689](https://issues.apache.org/jira/browse/HADOOP-15689) | Add "\*.patch" into .gitignore file of branch-2 |  Major | . | Rui Gao | Rui Gao |
+| [HDFS-13854](https://issues.apache.org/jira/browse/HDFS-13854) | RBF: The ProcessingAvgTime and ProxyAvgTime should display by JMX with ms unit. |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [YARN-8051](https://issues.apache.org/jira/browse/YARN-8051) | TestRMEmbeddedElector#testCallbackSynchronization is flakey |  Major | test | Robert Kanter | Robert Kanter |
+| [HDFS-13857](https://issues.apache.org/jira/browse/HDFS-13857) | RBF: Choose to enable the default nameservice to read/write files |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [HDFS-13812](https://issues.apache.org/jira/browse/HDFS-13812) | Fix the inconsistent default refresh interval on Caching documentation |  Trivial | documentation | BELUGA BEHR | Hrishikesh Gadre |
+| [HDFS-13902](https://issues.apache.org/jira/browse/HDFS-13902) |  Add JMX, conf and stacks menus to the datanode page |  Minor | datanode | fengchuang | fengchuang |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-15121](https://issues.apache.org/jira/browse/HADOOP-15121) | Encounter NullPointerException when using DecayRpcScheduler |  Major | . | Tao Jie | Tao Jie |
+| [YARN-7765](https://issues.apache.org/jira/browse/YARN-7765) | [Atsv2] GSSException: No valid credentials provided - Failed to find any Kerberos tgt thrown by Timelinev2Client & HBaseClient in NM |  Blocker | . | Sumana Sathish | Rohith Sharma K S |
+| [MAPREDUCE-7027](https://issues.apache.org/jira/browse/MAPREDUCE-7027) | HadoopArchiveLogs shouldn't delete the original logs if the HAR creation fails |  Critical | harchive | Gergely Novák | Gergely Novák |
+| [HDFS-10803](https://issues.apache.org/jira/browse/HDFS-10803) | TestBalancerWithMultipleNameNodes#testBalancing2OutOf3Blockpools fails intermittently due to no free space available |  Major | . | Yiqun Lin | Yiqun Lin |
+| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ |  Blocker | yarn | Sunil Govindan | Sunil Govindan |
+| [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop |  Major | . | Xiao Chen | Xiao Chen |
+| [HADOOP-15375](https://issues.apache.org/jira/browse/HADOOP-15375) | Branch-2 pre-commit failed to build docker image |  Major | . | Xiao Chen | Xiao Chen |
+| [HADOOP-15357](https://issues.apache.org/jira/browse/HADOOP-15357) | Configuration.getPropsWithPrefix no longer does variable substitution |  Major | . | Jim Brennan | Jim Brennan |
+| [HDFS-7101](https://issues.apache.org/jira/browse/HDFS-7101) | Potential null dereference in DFSck#doWork() |  Minor | . | Ted Yu | skrho |
+| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger |  Major | nodemanager, resourcemanager | Jason Lowe | Jason Lowe |
+| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails |  Major | test | Jason Lowe | Jason Lowe |
+| [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option |  Minor | . | Erik Krogen | Erik Krogen |
+| [HDFS-12828](https://issues.apache.org/jira/browse/HDFS-12828) | OIV ReverseXML Processor fails with escaped characters |  Critical | hdfs | Erik Krogen | Erik Krogen |
+| [HADOOP-15180](https://issues.apache.org/jira/browse/HADOOP-15180) | branch-2 : daemon processes' sysout overwrites 'ulimit -a' in daemon's out file |  Minor | scripts | Ranith Sardar | Ranith Sardar |
+| [HADOOP-15396](https://issues.apache.org/jira/browse/HADOOP-15396) | Some java source files are executable |  Minor | . | Akira Ajisaka | Shashikant Banerjee |
+| [YARN-7786](https://issues.apache.org/jira/browse/YARN-7786) | NullPointerException while launching ApplicationMaster |  Major | . | lujie | lujie |
+| [HDFS-10183](https://issues.apache.org/jira/browse/HDFS-10183) | Prevent race condition during class initialization |  Minor | fs | Pavel Avgustinov | Pavel Avgustinov |
+| [HDFS-13408](https://issues.apache.org/jira/browse/HDFS-13408) | MiniDFSCluster to support being built on randomized base directory |  Major | test | Xiao Liang | Xiao Liang |
+| [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens |  Critical | . | Xiao Chen | Xiao Chen |
+| [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows |  Major | . | Xiao Liang | Xiao Liang |
+| [HADOOP-15385](https://issues.apache.org/jira/browse/HADOOP-15385) | Many tests are failing in hadoop-distcp project in branch-2 |  Critical | tools/distcp | Rushabh S Shah | Jason Lowe |
+| [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
+| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal |  Major | . | Bibin A Chundatt | Bibin A Chundatt |
+| [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens |  Major | resourcemanager | Hu Ziqian | Hu Ziqian |
+| [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows |  Major | . | Xiao Liang | Xiao Liang |
+| [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | WASB: PageBlobInputStream.skip breaks HBASE replication |  Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [YARN-7003](https://issues.apache.org/jira/browse/YARN-7003) | DRAINING state of queues is not recovered after RM restart |  Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) |  TestContainerSchedulerQueuing.testStartMultipleContainers failed |  Major | . | Miklos Szegedi | Jim Brennan |
+| [HDFS-13581](https://issues.apache.org/jira/browse/HDFS-13581) | DN UI logs link is broken when https is enabled |  Minor | datanode | Namit Maheshwari | Shashikant Banerjee |
+| [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows |  Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak |
+| [HDFS-13590](https://issues.apache.org/jira/browse/HDFS-13590) | Backport HDFS-12378 to branch-2 |  Major | datanode, hdfs, test | Lukas Majercak | Lukas Majercak |
+| [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation |  Blocker | . | Kihwal Lee | Arpit Agarwal |
+| [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows |  Major | . | Xiao Liang | Xiao Liang |
+| [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync |  Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows |  Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" |  Blocker | . | Rohith Sharma K S | Jason Lowe |
+| [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows |  Major | test | Xiao Liang | Xiao Liang |
+| [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 |  Critical | kms | Gabor Bota | Gabor Bota |
+| [MAPREDUCE-7103](https://issues.apache.org/jira/browse/MAPREDUCE-7103) | Fix TestHistoryViewerPrinter on windows due to a mismatch line separator |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-8359](https://issues.apache.org/jira/browse/YARN-8359) | Exclude containermanager.linux test classes on Windows |  Major | . | Giovanni Matteo Fumarola | Jason Lowe |
+| [HDFS-13664](https://issues.apache.org/jira/browse/HDFS-13664) | Refactor ConfiguredFailoverProxyProvider to make inheritance easier |  Minor | hdfs-client | Chao Sun | Chao Sun |
+| [HDFS-13667](https://issues.apache.org/jira/browse/HDFS-13667) | Typo: Marking all "datandoes" as stale |  Trivial | namenode | Wei-Chiu Chuang | Nanda kumar |
+| [YARN-8405](https://issues.apache.org/jira/browse/YARN-8405) | RM zk-state-store.parent-path ACLs has been changed since HADOOP-14773 |  Major | . | Rohith Sharma K S | Íñigo Goiri |
+| [MAPREDUCE-7108](https://issues.apache.org/jira/browse/MAPREDUCE-7108) | TestFileOutputCommitter fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
+| [YARN-8404](https://issues.apache.org/jira/browse/YARN-8404) | Timeline event publish need to be async to avoid Dispatcher thread leak in case ATS is down |  Blocker | . | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-13675](https://issues.apache.org/jira/browse/HDFS-13675) | Speed up TestDFSAdminWithHA |  Major | hdfs, namenode | Lukas Majercak | Lukas Majercak |
+| [HDFS-13673](https://issues.apache.org/jira/browse/HDFS-13673) | TestNameNodeMetrics fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
+| [HDFS-13676](https://issues.apache.org/jira/browse/HDFS-13676) | TestEditLogRace fails on Windows |  Minor | test | Zuoming Zhang | Zuoming Zhang |
+| [HADOOP-15523](https://issues.apache.org/jira/browse/HADOOP-15523) | Shell command timeout given is in seconds whereas it is taken as millisec while scheduling |  Major | . | Bilwa S T | Bilwa S T |
+| [YARN-8444](https://issues.apache.org/jira/browse/YARN-8444) | NodeResourceMonitor crashes on bad swapFree value |  Major | . | Jim Brennan | Jim Brennan |
+| [YARN-8443](https://issues.apache.org/jira/browse/YARN-8443) | Total #VCores in cluster metrics is wrong when CapacityScheduler reserved some containers |  Major | webapp | Tao Yang | Tao Yang |
+| [YARN-8457](https://issues.apache.org/jira/browse/YARN-8457) | Compilation is broken with -Pyarn-ui |  Major | webapp | Sunil Govindan | Sunil Govindan |
+| [YARN-8401](https://issues.apache.org/jira/browse/YARN-8401) | [UI2] new ui is not accessible with out internet connection |  Blocker | . | Bibin A Chundatt | Bibin A Chundatt |
+| [YARN-8451](https://issues.apache.org/jira/browse/YARN-8451) | Multiple NM heartbeat thread created when a slow NM resync with RM |  Major | nodemanager | Botong Huang | Botong Huang |
+| [HADOOP-15548](https://issues.apache.org/jira/browse/HADOOP-15548) | Randomize local dirs |  Minor | . | Jim Brennan | Jim Brennan |
+| [YARN-8473](https://issues.apache.org/jira/browse/YARN-8473) | Containers being launched as app tears down can leave containers in NEW state |  Major | nodemanager | Jason Lowe | Jason Lowe |
+| [HDFS-13729](https://issues.apache.org/jira/browse/HDFS-13729) | Fix broken links to RBF documentation |  Minor | documentation | jwhitter | Gabor Bota |
+| [YARN-8518](https://issues.apache.org/jira/browse/YARN-8518) | test-container-executor test\_is\_empty() is broken |  Major | . | Jim Brennan | Jim Brennan |
+| [YARN-8515](https://issues.apache.org/jira/browse/YARN-8515) | container-executor can crash with SIGPIPE after nodemanager restart |  Major | . | Jim Brennan | Jim Brennan |
+| [YARN-8421](https://issues.apache.org/jira/browse/YARN-8421) | when moving app, activeUsers is increased, even though app does not have outstanding request |  Major | . | kyungwan nam |  |
+| [HADOOP-15614](https://issues.apache.org/jira/browse/HADOOP-15614) | TestGroupsCaching.testExceptionOnBackgroundRefreshHandled reliably fails |  Major | . | Kihwal Lee | Weiwei Yang |
+| [YARN-8577](https://issues.apache.org/jira/browse/YARN-8577) | Fix the broken anchor in SLS site-doc |  Minor | documentation | Weiwei Yang | Weiwei Yang |
+| [YARN-4606](https://issues.apache.org/jira/browse/YARN-4606) | CapacityScheduler: applications could get starved because computation of #activeUsers considers pending apps |  Critical | capacity scheduler, capacityscheduler | Karam Singh | Manikandan R |
+| [HADOOP-15637](https://issues.apache.org/jira/browse/HADOOP-15637) | LocalFs#listLocatedStatus does not filter out hidden .crc files |  Minor | fs | Erik Krogen | Erik Krogen |
+| [HADOOP-15644](https://issues.apache.org/jira/browse/HADOOP-15644) | Hadoop Docker Image Pip Install Fails on branch-2 |  Critical | build | Haibo Chen | Haibo Chen |
+| [YARN-8331](https://issues.apache.org/jira/browse/YARN-8331) | Race condition in NM container launched after done |  Major | . | Yang Wang | Pradeep Ambati |
+| [HDFS-13758](https://issues.apache.org/jira/browse/HDFS-13758) | DatanodeManager should throw exception if it has BlockRecoveryCommand but the block is not under construction |  Major | namenode | Wei-Chiu Chuang | chencan |
+| [YARN-8612](https://issues.apache.org/jira/browse/YARN-8612) | Fix NM Collector Service Port issue in YarnConfiguration |  Major | ATSv2 | Prabha Manepalli | Prabha Manepalli |
+| [HADOOP-15674](https://issues.apache.org/jira/browse/HADOOP-15674) | Test failure TestSSLHttpServer.testExcludedCiphers with TLS\_ECDHE\_RSA\_WITH\_AES\_128\_CBC\_SHA256 cipher suite |  Major | common | Gabor Bota | Szilard Nemeth |
+| [YARN-8640](https://issues.apache.org/jira/browse/YARN-8640) | Restore previous state in container-executor after failure |  Major | . | Jim Brennan | Jim Brennan |
+| [HADOOP-14314](https://issues.apache.org/jira/browse/HADOOP-14314) | The OpenSolaris taxonomy link is dead in InterfaceClassification.md |  Major | documentation | Daniel Templeton | Rui Gao |
+| [YARN-8649](https://issues.apache.org/jira/browse/YARN-8649) | NPE in localizer hearbeat processing if a container is killed while localizing |  Major | . | lujie | lujie |
+| [HADOOP-10219](https://issues.apache.org/jira/browse/HADOOP-10219) | ipc.Client.setupIOstreams() needs to check for ClientCache.stopClient requested shutdowns |  Major | ipc | Steve Loughran | Kihwal Lee |
+| [MAPREDUCE-7131](https://issues.apache.org/jira/browse/MAPREDUCE-7131) | Job History Server has race condition where it moves files from intermediate to finished but thinks file is in intermediate |  Major | . | Anthony Hsu | Anthony Hsu |
+| [HDFS-13836](https://issues.apache.org/jira/browse/HDFS-13836) | RBF: Handle mount table znode with null value |  Major | federation, hdfs | yanghuafeng | yanghuafeng |
+| [YARN-8709](https://issues.apache.org/jira/browse/YARN-8709) | CS preemption monitor always fails since one under-served queue was deleted |  Major | capacityscheduler, scheduler preemption | Tao Yang | Tao Yang |
+| [HDFS-13051](https://issues.apache.org/jira/browse/HDFS-13051) | Fix dead lock during async editlog rolling if edit queue is full |  Major | namenode | zhangwei | Daryn Sharp |
+| [YARN-8729](https://issues.apache.org/jira/browse/YARN-8729) | Node status updater thread could be lost after it is restarted |  Critical | nodemanager | Tao Yang | Tao Yang |
+| [HDFS-13914](https://issues.apache.org/jira/browse/HDFS-13914) | Fix DN UI logs link broken when https is enabled after HDFS-13902 |  Minor | datanode | Jianfei Jiang | Jianfei Jiang |
+| [MAPREDUCE-7133](https://issues.apache.org/jira/browse/MAPREDUCE-7133) | History Server task attempts REST API returns invalid data |  Major | jobhistoryserver | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [YARN-8720](https://issues.apache.org/jira/browse/YARN-8720) | CapacityScheduler does not enforce max resource allocation check at queue level |  Major | capacity scheduler, capacityscheduler, resourcemanager | Tarun Parimi | Tarun Parimi |
+| [HDFS-13844](https://issues.apache.org/jira/browse/HDFS-13844) | Fix the fmt\_bytes function in the dfs-dust.js |  Minor | hdfs, ui | yanghuafeng | yanghuafeng |
+| [HADOOP-15755](https://issues.apache.org/jira/browse/HADOOP-15755) | StringUtils#createStartupShutdownMessage throws NPE when args is null |  Major | . | Lokesh Jain | Dinesh Chitlangia |
+| [MAPREDUCE-3801](https://issues.apache.org/jira/browse/MAPREDUCE-3801) | org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently |  Major | mrv2 | Robert Joseph Evans | Jason Lowe |
+| [MAPREDUCE-7137](https://issues.apache.org/jira/browse/MAPREDUCE-7137) | MRAppBenchmark.benchmark1() fails with NullPointerException |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [MAPREDUCE-7138](https://issues.apache.org/jira/browse/MAPREDUCE-7138) | ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler |  Minor | test | Oleksandr Shevchenko | Oleksandr Shevchenko |
+| [HDFS-13908](https://issues.apache.org/jira/browse/HDFS-13908) | TestDataNodeMultipleRegistrations is flaky |  Major | . | Íñigo Goiri | Ayush Saxena |
+| [YARN-8804](https://issues.apache.org/jira/browse/YARN-8804) | resourceLimits may be wrongly calculated when leaf-queue is blocked in cluster with 3+ level queues |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-8774](https://issues.apache.org/jira/browse/YARN-8774) | Memory leak when CapacityScheduler allocates from reserved container with non-default label |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-15817](https://issues.apache.org/jira/browse/HADOOP-15817) | Reuse Object Mapper in KMSJSONReader |  Major | kms | Jonathan Eagles | Jonathan Eagles |
+| [HADOOP-15820](https://issues.apache.org/jira/browse/HADOOP-15820) | ZStandardDecompressor native code sets an integer field as a long |  Blocker | . | Jason Lowe | Jason Lowe |
+| [HDFS-13964](https://issues.apache.org/jira/browse/HDFS-13964) | RBF: TestRouterWebHDFSContractAppend fails with No Active Namenode under nameservice |  Major | . | Ayush Saxena | Ayush Saxena |
+| [HADOOP-15835](https://issues.apache.org/jira/browse/HADOOP-15835) | Reuse Object Mapper in KMSJSONWriter |  Major | . | Jonathan Eagles | Jonathan Eagles |
+| [HDFS-13976](https://issues.apache.org/jira/browse/HDFS-13976) | Backport HDFS-12813 to branch-2.9 |  Major | hdfs, hdfs-client | Lukas Majercak | Lukas Majercak |
+| [HADOOP-15679](https://issues.apache.org/jira/browse/HADOOP-15679) | ShutdownHookManager shutdown time needs to be configurable & extended |  Major | util | Steve Loughran | Steve Loughran |
+| [HDFS-13802](https://issues.apache.org/jira/browse/HDFS-13802) | RBF: Remove FSCK from Router Web UI |  Major | . | Fei Hui | Fei Hui |
+| [HADOOP-15859](https://issues.apache.org/jira/browse/HADOOP-15859) | ZStandardDecompressor.c mistakes a class for an instance |  Blocker | . | Ben Lau | Jason Lowe |
+| [HADOOP-15850](https://issues.apache.org/jira/browse/HADOOP-15850) | CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0 |  Critical | tools/distcp | Ted Yu | Ted Yu |
+| [YARN-7502](https://issues.apache.org/jira/browse/YARN-7502) | Nodemanager restart docs should describe nodemanager supervised property |  Major | documentation | Jason Lowe | Suma Shivaprasad |
+| [HADOOP-15866](https://issues.apache.org/jira/browse/HADOOP-15866) | Renamed HADOOP\_SECURITY\_GROUP\_SHELL\_COMMAND\_TIMEOUT keys break compatibility |  Blocker | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-15822](https://issues.apache.org/jira/browse/HADOOP-15822) | zstd compressor can fail with a small output buffer |  Major | . | Jason Lowe | Jason Lowe |
+| [HADOOP-15899](https://issues.apache.org/jira/browse/HADOOP-15899) | Update AWS Java SDK versions in NOTICE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-15900](https://issues.apache.org/jira/browse/HADOOP-15900) | Update JSch versions in LICENSE.txt |  Major | . | Akira Ajisaka | Akira Ajisaka |
+| [YARN-8858](https://issues.apache.org/jira/browse/YARN-8858) | CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. |  Major | . | Sumana Sathish | Wangda Tan |
+| [YARN-8233](https://issues.apache.org/jira/browse/YARN-8233) | NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null |  Critical | capacityscheduler | Tao Yang | Tao Yang |
+| [HADOOP-15923](https://issues.apache.org/jira/browse/HADOOP-15923) | create-release script should set max-cache-ttl as well as default-cache-ttl for gpg-agent |  Blocker | build | Akira Ajisaka | Akira Ajisaka |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13337](https://issues.apache.org/jira/browse/HDFS-13337) | Backport HDFS-4275 to branch-2.9 |  Minor | . | Íñigo Goiri | Xiao Liang |
+| [HDFS-13503](https://issues.apache.org/jira/browse/HDFS-13503) | Fix TestFsck test failures on Windows |  Major | hdfs | Xiao Liang | Xiao Liang |
+| [HDFS-13542](https://issues.apache.org/jira/browse/HDFS-13542) | TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13551](https://issues.apache.org/jira/browse/HDFS-13551) | TestMiniDFSCluster#testClusterSetStorageCapacity does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-11700](https://issues.apache.org/jira/browse/HDFS-11700) | TestHDFSServerPorts#testBackupNodePorts doesn't pass on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13548](https://issues.apache.org/jira/browse/HDFS-13548) | TestResolveHdfsSymlink#testFcResolveAfs fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13567](https://issues.apache.org/jira/browse/HDFS-13567) | TestNameNodeMetrics#testGenerateEDEKTime,TestNameNodeMetrics#testResourceCheck should use a different cluster basedir |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13557](https://issues.apache.org/jira/browse/HDFS-13557) | TestDFSAdmin#testListOpenFiles fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13550](https://issues.apache.org/jira/browse/HDFS-13550) | TestDebugAdmin#testComputeMetaCommand fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13559](https://issues.apache.org/jira/browse/HDFS-13559) | TestBlockScanner does not close TestContext properly |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13570](https://issues.apache.org/jira/browse/HDFS-13570) | TestQuotaByStorageType,TestQuota,TestDFSOutputStream fail on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13558](https://issues.apache.org/jira/browse/HDFS-13558) | TestDatanodeHttpXFrame does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13554](https://issues.apache.org/jira/browse/HDFS-13554) | TestDatanodeRegistration#testForcedRegistration does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13556](https://issues.apache.org/jira/browse/HDFS-13556) | TestNestedEncryptionZones does not shut down cluster |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13560](https://issues.apache.org/jira/browse/HDFS-13560) | Insufficient system resources exist to complete the requested service for some tests on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13592](https://issues.apache.org/jira/browse/HDFS-13592) | TestNameNodePrunesMissingStorages#testNameNodePrunesUnreportedStorages does not shut down cluster properly |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13593](https://issues.apache.org/jira/browse/HDFS-13593) | TestBlockReaderLocalLegacy#testBlockReaderLocalLegacyWithAppend fails on Windows |  Minor | test | Anbang Hu | Anbang Hu |
+| [HDFS-13587](https://issues.apache.org/jira/browse/HDFS-13587) | TestQuorumJournalManager fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13620](https://issues.apache.org/jira/browse/HDFS-13620) | Randomize the test directory path for TestHDFSFileSystemContract |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13591](https://issues.apache.org/jira/browse/HDFS-13591) | TestDFSShell#testSetrepLow fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13632](https://issues.apache.org/jira/browse/HDFS-13632) | Randomize baseDir for MiniJournalCluster in MiniQJMHACluster for TestDFSAdminWithHA |  Minor | . | Anbang Hu | Anbang Hu |
+| [MAPREDUCE-7102](https://issues.apache.org/jira/browse/MAPREDUCE-7102) | Fix TestJavaSerialization for Windows due a mismatch line separator |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [HDFS-13652](https://issues.apache.org/jira/browse/HDFS-13652) | Randomize baseDir for MiniDFSCluster in TestBlockScanner |  Minor | . | Anbang Hu | Anbang Hu |
+| [YARN-8370](https://issues.apache.org/jira/browse/YARN-8370) | Some Node Manager tests fail on Windows due to improper path/file separator |  Minor | . | Anbang Hu | Anbang Hu |
+| [YARN-8422](https://issues.apache.org/jira/browse/YARN-8422) | TestAMSimulator failing with NPE |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [HADOOP-15532](https://issues.apache.org/jira/browse/HADOOP-15532) | TestBasicDiskValidator fails with NoSuchFileException |  Minor | . | Íñigo Goiri | Giovanni Matteo Fumarola |
+| [HDFS-13563](https://issues.apache.org/jira/browse/HDFS-13563) | TestDFSAdminWithHA times out on Windows |  Minor | . | Anbang Hu | Lukas Majercak |
+| [HDFS-13681](https://issues.apache.org/jira/browse/HDFS-13681) | Fix TestStartup.testNNFailToStartOnReadOnlyNNDir test failure on Windows |  Major | test | Xiao Liang | Xiao Liang |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-13299](https://issues.apache.org/jira/browse/HDFS-13299) | RBF : Fix compilation error in branch-2 (TestMultipleDestinationResolver) |  Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed |  Major | test | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-8110](https://issues.apache.org/jira/browse/YARN-8110) | AMRMProxy recover should catch for all throwable to avoid premature exit |  Major | . | Botong Huang | Botong Huang |
+| [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix  java doc for StateStoreFileSystemImpl |  Minor | hdfs | Yiran Wu | Yiran Wu |
+| [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster |  Minor | . | Wei Yan | Íñigo Goiri |
+| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java |  Trivial | federation | BELUGA BEHR | BELUGA BEHR |
+| [HDFS-13386](https://issues.apache.org/jira/browse/HDFS-13386) | RBF: Wrong date information in list file(-ls) result |  Minor | . | Dibyendu Karmakar | Dibyendu Karmakar |
+| [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user |  Major | . | Shane Kumpf | Shane Kumpf |
+| [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace |  Major | . | Yiqun Lin | Yiqun Lin |
+| [YARN-7189](https://issues.apache.org/jira/browse/YARN-7189) | Container-executor doesn't remove Docker containers that error out early |  Major | yarn | Eric Badger | Eric Badger |
+| [HDFS-13466](https://issues.apache.org/jira/browse/HDFS-13466) | RBF: Add more router-related information to the UI |  Minor | . | Wei Yan | Wei Yan |
+| [HDFS-13453](https://issues.apache.org/jira/browse/HDFS-13453) | RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table |  Major | . | Dibyendu Karmakar | Dibyendu Karmakar |
+| [HDFS-13478](https://issues.apache.org/jira/browse/HDFS-13478) | RBF: Disabled Nameservice store API |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13490](https://issues.apache.org/jira/browse/HDFS-13490) | RBF: Fix setSafeMode in the Router |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13484](https://issues.apache.org/jira/browse/HDFS-13484) | RBF: Disable Nameservices from the federation |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13326](https://issues.apache.org/jira/browse/HDFS-13326) | RBF: Improve the interfaces to modify and view mount tables |  Minor | . | Wei Yan | Gang Li |
+| [HDFS-13499](https://issues.apache.org/jira/browse/HDFS-13499) | RBF: Show disabled name services in the UI |  Minor | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13508](https://issues.apache.org/jira/browse/HDFS-13508) | RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries |  Minor | . | Ekanth Sethuramalingam | Ekanth Sethuramalingam |
+| [HDFS-13434](https://issues.apache.org/jira/browse/HDFS-13434) | RBF: Fix dead links in RBF document |  Major | documentation | Akira Ajisaka | Chetna Chaudhari |
+| [HDFS-13488](https://issues.apache.org/jira/browse/HDFS-13488) | RBF: Reject requests when a Router is overloaded |  Major | . | Íñigo Goiri | Íñigo Goiri |
+| [HDFS-13525](https://issues.apache.org/jira/browse/HDFS-13525) | RBF: Add unit test TestStateStoreDisabledNameservice |  Major | . | Yiqun Lin | Yiqun Lin |
+| [YARN-8253](https://issues.apache.org/jira/browse/YARN-8253) | HTTPS Ats v2 api call fails with "bad HTTP parsed" |  Critical | ATSv2 | Yesha Vora | Charan Hebri |
+| [HADOOP-15454](https://issues.apache.org/jira/browse/HADOOP-15454) | TestRollingFileSystemSinkWithLocal fails on Windows |  Major | test | Xiao Liang | Xiao Liang |
+| [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows |  Minor | . | Anbang Hu | Anbang Hu |
+| [HDFS-13637](https://issues.apache.org/jira/browse/HDFS-13637) | RBF: Router fails when threadIndex (in ConnectionPool) wraps around Integer.MIN\_VALUE |  Critical | federation | CR Hota | CR Hota |
+| [YARN-4781](https://issues.apache.org/jira/browse/YARN-4781) | Support intra-queue preemption for fairness ordering policy. |  Major | scheduler | Wangda Tan | Eric Payne |
+| [HDFS-13281](https://issues.apache.org/jira/browse/HDFS-13281) | Namenode#createFile should be /.reserved/raw/ aware. |  Critical | encryption | Rushabh S Shah | Rushabh S Shah |
+| [YARN-4677](https://issues.apache.org/jira/browse/YARN-4677) | RMNodeResourceUpdateEvent update from scheduler can lead to race condition |  Major | graceful, resourcemanager, scheduler | Brook Zhou | Wilfred Spiegelenburg |
+| [HADOOP-15529](https://issues.apache.org/jira/browse/HADOOP-15529) | ContainerLaunch#testInvalidEnvVariableSubstitutionType is not supported in Windows |  Minor | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [HADOOP-15458](https://issues.apache.org/jira/browse/HADOOP-15458) | TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows |  Minor | test | Xiao Liang | Xiao Liang |
+| [YARN-8481](https://issues.apache.org/jira/browse/YARN-8481) | AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters |  Minor | amrmproxy, federation | Botong Huang | Botong Huang |
+| [HDFS-13475](https://issues.apache.org/jira/browse/HDFS-13475) | RBF: Admin cannot enforce Router enter SafeMode |  Major | . | Wei Yan | Chao Sun |
+| [HDFS-13733](https://issues.apache.org/jira/browse/HDFS-13733) | RBF: Add Web UI configurations and descriptions to RBF document |  Minor | documentation | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13743](https://issues.apache.org/jira/browse/HDFS-13743) | RBF: Router throws NullPointerException due to the invalid initialization of MountTableResolver |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13750](https://issues.apache.org/jira/browse/HDFS-13750) | RBF: Router ID in RouterRpcClient is always null |  Major | . | Takanobu Asanuma | Takanobu Asanuma |
+| [HDFS-13848](https://issues.apache.org/jira/browse/HDFS-13848) | Refactor NameNode failover proxy providers |  Major | ha, hdfs-client | Konstantin Shvachko | Konstantin Shvachko |
+| [HADOOP-15699](https://issues.apache.org/jira/browse/HADOOP-15699) | Fix some of testContainerManager failures in Windows |  Major | . | Botong Huang | Botong Huang |
+| [HADOOP-15731](https://issues.apache.org/jira/browse/HADOOP-15731) | TestDistributedShell fails on Windows |  Major | . | Botong Huang | Botong Huang |
+| [HADOOP-15759](https://issues.apache.org/jira/browse/HADOOP-15759) | AliyunOSS: update oss-sdk version to 3.0.0 |  Major | fs/oss | wujinhu | wujinhu |
+| [HADOOP-15671](https://issues.apache.org/jira/browse/HADOOP-15671) | AliyunOSS: Support Assume Roles in AliyunOSS |  Major | fs/oss | wujinhu | wujinhu |
+| [HDFS-13790](https://issues.apache.org/jira/browse/HDFS-13790) | RBF: Move ClientProtocol APIs to its own module |  Major | . | Íñigo Goiri | Chao Sun |
+| [HADOOP-15607](https://issues.apache.org/jira/browse/HADOOP-15607) | AliyunOSS: fix duplicated partNumber issue in AliyunOSSBlockOutputStream |  Critical | . | wujinhu | wujinhu |
+| [HADOOP-15868](https://issues.apache.org/jira/browse/HADOOP-15868) | AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy |  Major | fs/oss | wujinhu | wujinhu |
+
+
+### OTHER:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d539c55/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
new file mode 100644
index 0000000..5faa4cf
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.9.2/RELEASENOTES.2.9.2.md
@@ -0,0 +1,31 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop"  2.9.2 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HADOOP-15446](https://issues.apache.org/jira/browse/HADOOP-15446) | *Major* | **WASB: PageBlobInputStream.skip breaks HBASE replication**
+
+WASB: Bug fix to support non-sequential page blob reads.  Required for HBASE replication.
+
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d539c55/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
new file mode 100644
index 0000000..8ca51fa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.9.2.xml
@@ -0,0 +1,312 @@
+<?xml version="1.0" encoding="iso-8859-1" standalone="no"?>
+<!-- Generated by the JDiff Javadoc doclet -->
+<!-- (http://www.jdiff.org) -->
+<!-- on Tue Nov 13 16:20:19 UTC 2018 -->
+
+<api
+  xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+  xsi:noNamespaceSchemaLocation='api.xsd'
+  name="Apache Hadoop HDFS 2.9.2"
+  jdversion="1.0.9">
+
+<!--  Command line arguments =  -doclet org.apache.hadoop.classification.tools.IncludePublicAnnotationsJDiffDoclet -docletpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-annotations.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs/target/jdiff.jar -verbose -classpath /build/source/hadoop-hdfs-project/hadoop-hdfs/target/classes:/build/source/hadoop-common-project/hadoop-annotations/target/hadoop-annotations-2.9.2.jar:/usr/lib/jvm/java-7-openjdk-amd64/lib/tools.jar:/build/source/hadoop-common-project/hadoop-auth/target/hadoop-auth-2.9.2.jar:/maven/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar:/maven/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar:/maven/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar:/maven/com/nimbusds/nimbus-jose-jwt/4.41.1/nimbus-jose-jwt-4.41.1.jar:/maven/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar:/maven/net/minidev/json-smart/1.3.1/json-smart-1.3.1.jar:/maven/org/apache/directory/serv
 er/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar:/maven/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar:/maven/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar:/maven/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar:/maven/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar:/maven/jline/jline/0.9.94/jline-0.9.94.jar:/maven/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar:/build/source/hadoop-common-project/hadoop-common/target/hadoop-common-2.9.2.jar:/maven/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar:/maven/commons-net/commons-net/3.1/commons-net-3.1.jar:/maven/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar:/maven/org/mortbay/jetty/jetty-sslengine/6.1.26/jetty-sslengine-6.1.26.jar:/maven/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar:/maven/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar:/maven/org/codehaus
 /jettison/jettison/1.1/jettison-1.1.jar:/maven/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar:/maven/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/maven/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/maven/javax/activation/activation/1.1/activation-1.1.jar:/maven/org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar:/maven/org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar:/maven/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/maven/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/maven/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/maven/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/maven/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/maven/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/maven/org/apache/commons/commons-lang3/3.4/commons-lang3-3.4.jar:/maven/org/apache/avro/avro/1.7.7/avro-1.7.7.jar:/maven/com/thoug
 htworks/paranamer/paranamer/2.3/paranamer-2.3.jar:/maven/org/xerial/snappy/snappy-java/1.0.5/snappy-java-1.0.5.jar:/maven/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar:/maven/com/jcraft/jsch/0.1.54/jsch-0.1.54.jar:/maven/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar:/maven/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar:/maven/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar:/maven/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/maven/org/tukaani/xz/1.0/xz-1.0.jar:/maven/org/codehaus/woodstox/stax2-api/3.1.4/stax2-api-3.1.4.jar:/maven/com/fasterxml/woodstox/woodstox-core/5.0.3/woodstox-core-5.0.3.jar:/build/source/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-2.9.2.jar:/maven/com/squareup/okhttp/okhttp/2.7.5/okhttp-2.7.5.jar:/maven/com/squareup/okio/okio/1.6.0/okio-1.6.0.jar:/maven/com/google/guava/guava/11.0.2/guava-11.0.2.jar:/maven/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar:/maven/org/mortb
 ay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/maven/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/maven/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar:/maven/asm/asm/3.2/asm-3.2.jar:/maven/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/maven/commons-codec/commons-codec/1.4/commons-codec-1.4.jar:/maven/commons-io/commons-io/2.4/commons-io-2.4.jar:/maven/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/maven/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar:/maven/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar:/maven/log4j/log4j/1.2.17/log4j-1.2.17.jar:/maven/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/maven/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar:/maven/org/slf4j/slf4j-log4j12/1.7.25/slf4j-log4j12-1.7.25.jar:/maven/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar:/maven/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar:/maven/xmlenc/xmlenc/0.52/xmlenc
 -0.52.jar:/maven/io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar:/maven/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/maven/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar:/maven/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar:/maven/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/maven/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar:/maven/com/fasterxml/jackson/core/jackson-databind/2.7.8/jackson-databind-2.7.8.jar:/maven/com/fasterxml/jackson/core/jackson-annotations/2.7.8/jackson-annotations-2.7.8.jar:/maven/com/fasterxml/jackson/core/jackson-core/2.7.8/jackson-core-2.7.8.jar -sourcepath /build/source/hadoop-hdfs-project/hadoop-hdfs/src/main/java -apidir /build/source/hadoop-hdfs-project/hadoop-hdfs/target/site/jdiff/xml -apiname Apache Hadoop HDFS 2.9.2 -->
+<package name="org.apache.hadoop.hdfs">
+  <doc>
+  <![CDATA[<p>A distributed implementation of {@link
+org.apache.hadoop.fs.FileSystem}.  This is loosely modelled after
+Google's <a href="http://research.google.com/archive/gfs.html">GFS</a>.</p>
+
+<p>The most important difference is that unlike GFS, Hadoop DFS files 
+have strictly one writer at any one time.  Bytes are always appended 
+to the end of the writer's stream.  There is no notion of "record appends"
+or "mutations" that are then checked or reordered.  Writers simply emit 
+a byte stream.  That byte stream is guaranteed to be stored in the 
+order written.</p>]]>
+  </doc>
+</package>
+<package name="org.apache.hadoop.hdfs.net">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer">
+</package>
+<package name="org.apache.hadoop.hdfs.protocol.datatransfer.sasl">
+</package>
+<package name="org.apache.hadoop.hdfs.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.client">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.protocolPB">
+</package>
+<package name="org.apache.hadoop.hdfs.qjournal.server">
+  <!-- start interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+  <interface name="JournalNodeMXBean"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="getJournalsStatus" return="java.lang.String"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Get status information (e.g., whether formatted) of JournalNode's journals.
+ 
+ @return A string presenting status for each journal]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[This is the JMX management interface for JournalNode information]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.qjournal.server.JournalNodeMXBean -->
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.block">
+</package>
+<package name="org.apache.hadoop.hdfs.security.token.delegation">
+</package>
+<package name="org.apache.hadoop.hdfs.server.balancer">
+</package>
+<package name="org.apache.hadoop.hdfs.server.blockmanagement">
+</package>
+<package name="org.apache.hadoop.hdfs.server.common">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.fsdataset.impl">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web">
+</package>
+<package name="org.apache.hadoop.hdfs.server.datanode.web.webhdfs">
+</package>
+<package name="org.apache.hadoop.hdfs.server.mover">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode">
+  <!-- start interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <interface name="AuditLogger"    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <method name="initialize"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="conf" type="org.apache.hadoop.conf.Configuration"/>
+      <doc>
+      <![CDATA[Called during initialization of the logger.
+
+ @param conf The configuration object.]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <doc>
+      <![CDATA[Called to log an audit event.
+ <p>
+ This method must return as quickly as possible, since it's called
+ in a critical section of the NameNode's operation.
+
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's
+             metadata (permissions, owner, times, etc).]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Interface defining an audit logger.]]>
+    </doc>
+  </interface>
+  <!-- end interface org.apache.hadoop.hdfs.server.namenode.AuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <class name="HdfsAuditLogger" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <implements name="org.apache.hadoop.hdfs.server.namenode.AuditLogger"/>
+    <constructor name="HdfsAuditLogger"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="status" type="org.apache.hadoop.fs.FileStatus"/>
+    </method>
+    <method name="logAuditEvent"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="callerContext" type="org.apache.hadoop.ipc.CallerContext"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String, String,
+ FileStatus)} with additional parameters related to logging delegation token
+ tracking IDs.
+ 
+ @param succeeded Whether authorization succeeded.
+ @param userName Name of the user executing the request.
+ @param addr Remote address of the request.
+ @param cmd The requested command.
+ @param src Path of affected source file.
+ @param dst Path of affected destination file (if any).
+ @param stat File information for operations that change the file's metadata
+          (permissions, owner, times, etc).
+ @param callerContext Context information of the caller
+ @param ugi UserGroupInformation of the current user, or null if not logging
+          token tracking information
+ @param dtSecretManager The token secret manager, or null if not logging
+          token tracking information]]>
+      </doc>
+    </method>
+    <method name="logAuditEvent"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="succeeded" type="boolean"/>
+      <param name="userName" type="java.lang.String"/>
+      <param name="addr" type="java.net.InetAddress"/>
+      <param name="cmd" type="java.lang.String"/>
+      <param name="src" type="java.lang.String"/>
+      <param name="dst" type="java.lang.String"/>
+      <param name="stat" type="org.apache.hadoop.fs.FileStatus"/>
+      <param name="ugi" type="org.apache.hadoop.security.UserGroupInformation"/>
+      <param name="dtSecretManager" type="org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager"/>
+      <doc>
+      <![CDATA[Same as
+ {@link #logAuditEvent(boolean, String, InetAddress, String, String,
+ String, FileStatus, CallerContext, UserGroupInformation,
+ DelegationTokenSecretManager)} without {@link CallerContext} information.]]>
+      </doc>
+    </method>
+    <doc>
+    <![CDATA[Extension of {@link AuditLogger}.]]>
+    </doc>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.HdfsAuditLogger -->
+  <!-- start class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+  <class name="INodeAttributeProvider" extends="java.lang.Object"
+    abstract="true"
+    static="false" final="false" visibility="public"
+    deprecated="not deprecated">
+    <constructor name="INodeAttributeProvider"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+    </constructor>
+    <method name="start"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Initialize the provider. This method is called at NameNode startup
+ time.]]>
+      </doc>
+    </method>
+    <method name="stop"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <doc>
+      <![CDATA[Shutdown the provider. This method is called at NameNode shutdown time.]]>
+      </doc>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="fullPath" type="java.lang.String"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="true" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="pathElements" type="java.lang.String[]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getAttributes" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="components" type="byte[][]"/>
+      <param name="inode" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributes"/>
+    </method>
+    <method name="getExternalAccessControlEnforcer" return="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"
+      abstract="false" native="false" synchronized="false"
+      static="false" final="false" visibility="public"
+      deprecated="not deprecated">
+      <param name="defaultEnforcer" type="org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider.AccessControlEnforcer"/>
+      <doc>
+      <![CDATA[Can be over-ridden by implementations to provide a custom Access Control
+ Enforcer that can provide an alternate implementation of the
+ default permission checking logic.
+ @param defaultEnforcer The Default AccessControlEnforcer
+ @return The AccessControlEnforcer to use]]>
+      </doc>
+    </method>
+  </class>
+  <!-- end class org.apache.hadoop.hdfs.server.namenode.INodeAttributeProvider -->
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.ha">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.metrics">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.top.window">
+</package>
+<package name="org.apache.hadoop.hdfs.server.namenode.web.resources">
+</package>
+<package name="org.apache.hadoop.hdfs.server.protocol">
+</package>
+<package name="org.apache.hadoop.hdfs.tools">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineEditsViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.offlineImageViewer">
+</package>
+<package name="org.apache.hadoop.hdfs.tools.snapshot">
+</package>
+<package name="org.apache.hadoop.hdfs.util">
+</package>
+<package name="org.apache.hadoop.hdfs.web">
+</package>
+<package name="org.apache.hadoop.hdfs.web.resources">
+</package>
+
+</api>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[08/50] [abbrv] hadoop git commit: YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne

Posted by jh...@apache.org.
YARN-7225. Add queue and partition info to RM audit log. Contributed by Eric Payne


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/611b5c76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/611b5c76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/611b5c76

Branch: refs/heads/branch-2-jhung-test
Commit: 611b5c763086676776934ff0a62fa70c91030711
Parents: 401e104
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Thu Nov 1 14:13:15 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../server/resourcemanager/ClientRMService.java | 12 ++-
 .../server/resourcemanager/RMAuditLogger.java   | 81 +++++++++++++++++---
 .../scheduler/common/fica/FiCaSchedulerApp.java | 19 ++++-
 .../scheduler/fair/FSAppAttempt.java            |  5 +-
 .../scheduler/fifo/FifoAppAttempt.java          | 10 ++-
 .../resourcemanager/TestRMAuditLogger.java      | 23 +++++-
 6 files changed, 129 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 5b4123d..a10413b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -558,7 +558,8 @@ public class ClientRMService extends AbstractService implements
       LOG.warn("Unable to get the current user.", ie);
       RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
           ie.getMessage(), "ClientRMService",
-          "Exception in submitting application", applicationId, callerContext);
+          "Exception in submitting application", applicationId, callerContext,
+          submissionContext.getQueue());
       throw RPCUtil.getRemoteException(ie);
     }
 
@@ -580,7 +581,8 @@ public class ClientRMService extends AbstractService implements
             ". Flow run should be a long integer", e);
         RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
             e.getMessage(), "ClientRMService",
-            "Exception in submitting application", applicationId);
+            "Exception in submitting application", applicationId,
+            submissionContext.getQueue());
         throw RPCUtil.getRemoteException(e);
       }
     }
@@ -639,12 +641,14 @@ public class ClientRMService extends AbstractService implements
       LOG.info("Application with id " + applicationId.getId() + 
           " submitted by user " + user);
       RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST,
-          "ClientRMService", applicationId, callerContext);
+          "ClientRMService", applicationId, callerContext,
+          submissionContext.getQueue());
     } catch (YarnException e) {
       LOG.info("Exception in submitting " + applicationId, e);
       RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
           e.getMessage(), "ClientRMService",
-          "Exception in submitting application", applicationId, callerContext);
+          "Exception in submitting application", applicationId, callerContext,
+          submissionContext.getQueue());
       throw e;
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
index 4f191a6..af66229 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
@@ -40,7 +40,7 @@ public class RMAuditLogger {
   static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
                     DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, 
                     CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME,
-                    INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
+                    INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE, NODELABEL}
 
   public static class AuditConstants {
     static final String SUCCESS = "SUCCESS";
@@ -98,7 +98,7 @@ public class RMAuditLogger {
       ApplicationId appId, ApplicationAttemptId attemptId,
       ContainerId containerId, Resource resource) {
     return createSuccessLog(user, operation, target, appId, attemptId,
-        containerId, resource, null, Server.getRemoteIp());
+        containerId, resource, null, Server.getRemoteIp(), null, null);
   }
 
   /**
@@ -124,7 +124,7 @@ public class RMAuditLogger {
   static String createSuccessLog(String user, String operation, String target,
       ApplicationId appId, ApplicationAttemptId attemptId,
       ContainerId containerId, Resource resource, CallerContext callerContext,
-      InetAddress ip) {
+      InetAddress ip, String queueName, String partition) {
     StringBuilder b =
         createStringBuilderForSuccessEvent(user, operation, target, ip);
     if (appId != null) {
@@ -140,6 +140,12 @@ public class RMAuditLogger {
       add(Keys.RESOURCE, resource.toString(), b);
     }
     appendCallerContext(b, callerContext);
+    if (queueName != null) {
+      add(Keys.QUEUENAME, queueName, b);
+    }
+    if (partition != null) {
+      add(Keys.NODELABEL, partition, b);
+    }
     return b.toString();
   }
   
@@ -203,6 +209,32 @@ public class RMAuditLogger {
   }
 
   /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user User who made the service request to the ResourceManager
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed.
+   * @param appId Application Id in which operation was performed.
+   * @param containerId Container Id in which operation was performed.
+   * @param resource Resource associated with container.
+   * @param queueName Name of queue.
+   * @param partition Name of labeled partition.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, ContainerId containerId, Resource resource,
+      String queueName, String partition) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null,
+          containerId, resource, null, Server.getRemoteIp(), queueName,
+          partition));
+    }
+  }
+
+  /**
    * Create a general readable and parseable audit log string for a successful
    * event.
    *
@@ -263,12 +295,20 @@ public class RMAuditLogger {
           null, null));
     }
   }
-  
+
   public static void logSuccess(String user, String operation, String target,
       ApplicationId appId, CallerContext callerContext) {
     if (LOG.isInfoEnabled()) {
       LOG.info(createSuccessLog(user, operation, target, appId, null, null,
-          null, callerContext, Server.getRemoteIp()));
+          null, callerContext, Server.getRemoteIp(), null, null));
+    }
+  }
+
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, CallerContext callerContext, String queueName) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, appId, null, null,
+          null, callerContext, Server.getRemoteIp(), queueName, null));
     }
   }
 
@@ -296,7 +336,7 @@ public class RMAuditLogger {
       ApplicationId appId, InetAddress ip) {
     if (LOG.isInfoEnabled()) {
       LOG.info(createSuccessLog(user, operation, target, appId, null, null,
-          null, null, ip));
+          null, null, ip, null, null));
     }
   }
 
@@ -355,7 +395,7 @@ public class RMAuditLogger {
   static String createFailureLog(String user, String operation, String perm,
       String target, String description, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId,
-      Resource resource, CallerContext callerContext) {
+      Resource resource, CallerContext callerContext, String queueName) {
     StringBuilder b = createStringBuilderForFailureLog(user,
         operation, target, description, perm);
     if (appId != null) {
@@ -371,6 +411,9 @@ public class RMAuditLogger {
       add(Keys.RESOURCE, resource.toString(), b);
     }
     appendCallerContext(b, callerContext);
+    if (queueName != null) {
+      add(Keys.QUEUENAME, queueName, b);
+    }
     return b.toString();
   }
 
@@ -381,7 +424,7 @@ public class RMAuditLogger {
       String target, String description, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) {
     return createFailureLog(user, operation, perm, target, description, appId,
-        attemptId, containerId, resource, null);
+        attemptId, containerId, resource, null, null);
   }
 
   /**
@@ -447,13 +490,22 @@ public class RMAuditLogger {
           appId, attemptId, null, null));
     }
   }
-  
+
   public static void logFailure(String user, String operation, String perm,
       String target, String description, ApplicationId appId,
       CallerContext callerContext) {
     if (LOG.isWarnEnabled()) {
       LOG.warn(createFailureLog(user, operation, perm, target, description,
-          appId, null, null, null, callerContext));
+          appId, null, null, null, callerContext, null));
+    }
+  }
+
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      CallerContext callerContext, String queueName) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, null, null, null, callerContext, queueName));
     }
   }
 
@@ -480,6 +532,15 @@ public class RMAuditLogger {
     }
   }
 
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      String queueName) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          appId, null, null, null, null, queueName));
+    }
+  }
+
   /**
    * Create a readable and parseable audit log string for a failed event.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index cab4dd9..e7bb027 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -191,9 +191,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
 
       containersToPreempt.remove(containerId);
 
+      // In order to save space in the audit log, only include the partition
+      // if it is not the default partition.
+      String containerPartition = null;
+      if (partition != null && !partition.isEmpty()) {
+        containerPartition = partition;
+      }
       Resource containerResource = rmContainer.getContainer().getResource();
       RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
-          "SchedulerApp", getApplicationId(), containerId, containerResource);
+          "SchedulerApp", getApplicationId(), containerId, containerResource,
+          getQueueName(), containerPartition);
 
       // Update usage metrics
       queue.getMetrics().releaseResources(partition,
@@ -566,9 +573,17 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
                 + " host=" + rmContainer.getAllocatedNode().getHost()
                 + " type=" + allocation.getAllocationLocalityType());
           }
+          // In order to save space in the audit log, only include the partition
+          // if it is not the default partition.
+          String partition =
+              schedulerContainer.getSchedulerNode().getPartition();
+          if (partition != null && partition.isEmpty()) {
+            partition = null;
+          }
           RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
               "SchedulerApp", getApplicationId(), containerId,
-              allocation.getAllocatedOrReservedResource());
+              allocation.getAllocatedOrReservedResource(), getQueueName(),
+              partition);
         } else {
           // If the rmContainer's state is already updated to RESERVED, this is
           // a reReservation

http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 21863b8..5897914 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -168,7 +168,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
 
       Resource containerResource = rmContainer.getContainer().getResource();
       RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
-          "SchedulerApp", getApplicationId(), containerId, containerResource);
+          "SchedulerApp", getApplicationId(), containerId, containerResource,
+          rmContainer.getQueueName(), null);
 
       // Update usage metrics
       queue.getMetrics().releaseResources(
@@ -486,7 +487,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
       }
       RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
           "SchedulerApp", getApplicationId(), container.getId(),
-          container.getResource());
+          container.getResource(), getQueueName(), null);
     } finally {
       writeLock.unlock();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
index d932e0e..230f66f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoAppAttempt.java
@@ -98,9 +98,17 @@ public class FifoAppAttempt extends FiCaSchedulerApp {
             .getApplicationAttemptId() + " container=" + containerId + " host="
             + container.getNodeId().getHost() + " type=" + type);
       }
+      // In order to save space in the audit log, only include the partition
+      // if it is not the default partition.
+      String partition = null;
+      if (appAMNodePartitionName != null &&
+            !appAMNodePartitionName.isEmpty()) {
+        partition = appAMNodePartitionName;
+      }
       RMAuditLogger.logSuccess(getUser(),
           RMAuditLogger.AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
-          getApplicationId(), containerId, container.getResource());
+          getApplicationId(), containerId, container.getResource(),
+          getQueueName(), partition);
 
       return rmContainer;
     } finally {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/611b5c76/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
index d5f1e35..7d8eeab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
@@ -67,6 +67,7 @@ public class TestRMAuditLogger {
   private static final Resource RESOURCE = mock(Resource.class);
   private static final String CALLER_CONTEXT = "context";
   private static final byte[] CALLER_SIGNATURE = "signature".getBytes();
+  private static final String PARTITION = "label1";
 
   @Before
   public void setUp() throws Exception {
@@ -132,6 +133,14 @@ public class TestRMAuditLogger {
         ApplicationAttemptId attemptId, ContainerId containerId,
         CallerContext callerContext, Resource resource, InetAddress remoteIp,
         RMAuditLogger.ArgsBuilder args) {
+    testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+        callerContext, resource, remoteIp, args, null, null);
+  }
+
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+        ApplicationAttemptId attemptId, ContainerId containerId,
+        CallerContext callerContext, Resource resource, InetAddress remoteIp,
+        RMAuditLogger.ArgsBuilder args, String queueName, String partition) {
     String sLog;
     InetAddress tmpIp = checkIP ? remoteIp : null;
     if (args != null) {
@@ -139,7 +148,8 @@ public class TestRMAuditLogger {
           tmpIp, args);
     } else {
       sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
-          attemptId, containerId, resource, callerContext, tmpIp);
+          attemptId, containerId, resource, callerContext, tmpIp, queueName,
+          partition);
     }
     StringBuilder expLog = new StringBuilder();
     expLog.append("USER=test\t");
@@ -177,6 +187,13 @@ public class TestRMAuditLogger {
     if (args != null) {
       expLog.append("\tQUEUENAME=root");
       expLog.append("\tRECURSIVE=true");
+    } else {
+      if (queueName != null) {
+        expLog.append("\tQUEUENAME=" + QUEUE);
+      }
+    }
+    if (partition != null) {
+      expLog.append("\tNODELABEL=" + PARTITION);
     }
     assertEquals(expLog.toString(), sLog);
   }
@@ -258,6 +275,8 @@ public class TestRMAuditLogger {
         .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
     testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
         Server.getRemoteIp(), args);
+    testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
+        Server.getRemoteIp(), null, QUEUE, PARTITION);
     testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
     testSuccessLogNulls(checkIP);
   }
@@ -283,7 +302,7 @@ public class TestRMAuditLogger {
         RMAuditLogger.ArgsBuilder args) {
     String fLog = args == null ?
       RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
-          appId, attemptId, containerId, resource, callerContext) :
+          appId, attemptId, containerId, resource, callerContext, null) :
         RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
             args);
     StringBuilder expLog = new StringBuilder();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[20/50] [abbrv] hadoop git commit: HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe.

Posted by jh...@apache.org.
HADOOP-15822. zstd compressor can fail with a small output buffer. Contributed by Jason Lowe.

(cherry picked from commit 8f97d6f2cdfccefba5457ae3d561e9ce0109da3f)

Conflicts:
	dev-support/docker/Dockerfile


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/645b3116
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/645b3116
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/645b3116

Branch: refs/heads/branch-2-jhung-test
Commit: 645b3116ad29f34b184b7e67bc19a0b4b5e716dc
Parents: b32ef14
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 24 13:49:28 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/io/compress/zstd/ZStandardDecompressor.java   |  4 ++--
 .../apache/hadoop/io/compress/zstd/ZStandardCompressor.c | 11 +++++++----
 .../hadoop/io/compress/zstd/ZStandardDecompressor.c      |  1 +
 .../zstd/TestZStandardCompressorDecompressor.java        | 10 ++++------
 4 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/645b3116/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
index 73d73e1..bc9d29c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.java
@@ -262,8 +262,8 @@ public class ZStandardDecompressor implements Decompressor {
 
     int originalPosition = dst.position();
     int n = inflateBytesDirect(
-        src, src.position(), src.remaining(), dst, dst.position(),
-        dst.remaining()
+        src, src.position(), src.limit(), dst, dst.position(),
+        dst.limit()
     );
     dst.position(originalPosition + n);
     if (bytesInCompressedBuffer > 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/645b3116/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
index 055683a..289554b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
@@ -195,10 +195,13 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
     ZSTD_inBuffer input = { uncompressed_bytes, uncompressed_direct_buf_len, uncompressed_direct_buf_off };
     ZSTD_outBuffer output = { compressed_bytes, compressed_direct_buf_len, 0 };
 
-    size_t size = dlsym_ZSTD_compressStream(stream, &output, &input);
-    if (dlsym_ZSTD_isError(size)) {
-        THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
-        return (jint) 0;
+    size_t size;
+    if (uncompressed_direct_buf_len != 0) {
+        size = dlsym_ZSTD_compressStream(stream, &output, &input);
+        if (dlsym_ZSTD_isError(size)) {
+            THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(size));
+            return (jint) 0;
+        }
     }
     if (finish && input.pos == input.size) {
         // end the stream, flush and  write the frame epilogue

http://git-wip-us.apache.org/repos/asf/hadoop/blob/645b3116/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
index b97dd17..e28359b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
@@ -178,6 +178,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
         return (jint) 0;
     }
     uncompressed_bytes = ((char*) uncompressed_bytes) + uncompressed_direct_buf_off;
+    uncompressed_direct_buf_len -= uncompressed_direct_buf_off;
 
     ZSTD_inBuffer input = { compressed_bytes, compressed_direct_buf_len, compressed_direct_buf_off };
     ZSTD_outBuffer output = { uncompressed_bytes, uncompressed_direct_buf_len, 0 };

http://git-wip-us.apache.org/repos/asf/hadoop/blob/645b3116/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
index 04def24..dcfb7e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/zstd/TestZStandardCompressorDecompressor.java
@@ -414,13 +414,11 @@ public class TestZStandardCompressorDecompressor {
     outBuf.clear();
     while (!decompressor.finished()) {
       decompressor.decompress(inBuf, outBuf);
-      if (outBuf.remaining() == 0) {
-        outBuf.flip();
-        while (outBuf.remaining() > 0) {
-          assertEquals(expected.get(), outBuf.get());
-        }
-        outBuf.clear();
+      outBuf.flip();
+      while (outBuf.remaining() > 0) {
+        assertEquals(expected.get(), outBuf.get());
       }
+      outBuf.clear();
     }
     outBuf.flip();
     while (outBuf.remaining() > 0) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[17/50] [abbrv] hadoop git commit: Addendum: Fix potential FSImage corruption. Contributed by Xiao Chen.

Posted by jh...@apache.org.
Addendum: Fix potential FSImage corruption. Contributed by Xiao Chen.

(cherry picked from commit dd70b1f53836961029767660398819a3b04eec58)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/401e1044
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/401e1044
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/401e1044

Branch: refs/heads/branch-2-jhung-test
Commit: 401e1044b0f3ad0ee5003a4226c733e5266132dc
Parents: 6d02100
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 31 17:01:49 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java    | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/401e1044/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 86b570d..2d88636 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -107,6 +107,8 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
         .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY);
     configurationPropsToSkipCompare.add(DFSConfigKeys
         .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY);
+    configurationPropsToSkipCompare.add(DFSConfigKeys
+        .DFS_IMAGE_EXPANDED_STRING_TABLES_KEY);
 
     // Allocate
     xmlPropsToSkipCompare = new HashSet<String>();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[06/50] [abbrv] hadoop git commit: YARN-7652. Handle AM register requests asynchronously in FederationInterceptor. Contributed by Botong Huang.

Posted by jh...@apache.org.
YARN-7652. Handle AM register requests asynchronously in FederationInterceptor. Contributed by Botong Huang.

(cherry picked from commit c3d22d3b4569b7f87af4ee4abfcc284deebe90de)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f50f9cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f50f9cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f50f9cb

Branch: refs/heads/branch-2-jhung-test
Commit: 9f50f9cb9a5c5e23bea194f2e4c57a85b1703f16
Parents: 0c463c2
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Oct 9 10:29:40 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../yarn/server/AMHeartbeatRequestHandler.java  |  10 +-
 .../hadoop/yarn/server/AMRMClientRelayer.java   |   6 +-
 .../server/uam/UnmanagedApplicationManager.java |  20 +-
 .../yarn/server/MockResourceManagerFacade.java  |  29 +-
 .../uam/TestUnmanagedApplicationManager.java    |  51 +++-
 .../amrmproxy/FederationInterceptor.java        | 291 +++++++------------
 .../amrmproxy/TestFederationInterceptor.java    |  16 +-
 .../TestableFederationInterceptor.java          |  26 +-
 8 files changed, 217 insertions(+), 232 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
index 380c216..1534354 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMHeartbeatRequestHandler.java
@@ -59,7 +59,7 @@ public class AMHeartbeatRequestHandler extends Thread {
   private int lastResponseId;
 
   public AMHeartbeatRequestHandler(Configuration conf,
-      ApplicationId applicationId) {
+      ApplicationId applicationId, AMRMClientRelayer rmProxyRelayer) {
     super("AMHeartbeatRequestHandler Heartbeat Handler Thread");
     this.setUncaughtExceptionHandler(
         new HeartBeatThreadUncaughtExceptionHandler());
@@ -69,6 +69,7 @@ public class AMHeartbeatRequestHandler extends Thread {
     this.conf = conf;
     this.applicationId = applicationId;
     this.requestQueue = new LinkedBlockingQueue<>();
+    this.rmProxyRelayer = rmProxyRelayer;
 
     resetLastResponseId();
   }
@@ -157,13 +158,6 @@ public class AMHeartbeatRequestHandler extends Thread {
   }
 
   /**
-   * Set the AMRMClientRelayer for RM connection.
-   */
-  public void setAMRMClientRelayer(AMRMClientRelayer relayer) {
-    this.rmProxyRelayer = relayer;
-  }
-
-  /**
    * Set the UGI for RM connection.
    */
   public void setUGI(UserGroupInformation ugi) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
index 790147c..5c7f7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
@@ -181,7 +181,11 @@ public class AMRMClientRelayer extends AbstractService
     this.amRegistrationRequest = registerRequest;
   }
 
-  public void setRMClient(ApplicationMasterProtocol client){
+  public String getRMIdentifier() {
+    return this.rmId;
+  }
+
+  public void setRMClient(ApplicationMasterProtocol client) {
     this.rmClient = client;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
index 608fbab..c762ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
@@ -134,10 +134,12 @@ public class UnmanagedApplicationManager {
     this.submitter = submitter;
     this.appNameSuffix = appNameSuffix;
     this.userUgi = null;
-    this.heartbeatHandler =
-        new AMHeartbeatRequestHandler(this.conf, this.applicationId);
+    // Relayer's rmClient will be set after the RM connection is created
     this.rmProxyRelayer =
         new AMRMClientRelayer(null, this.applicationId, rmName);
+    this.heartbeatHandler = createAMHeartbeatRequestHandler(this.conf,
+        this.applicationId, this.rmProxyRelayer);
+
     this.connectionInitiated = false;
     this.registerRequest = null;
     this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
@@ -150,6 +152,13 @@ public class UnmanagedApplicationManager {
         keepContainersAcrossApplicationAttempts;
   }
 
+  @VisibleForTesting
+  protected AMHeartbeatRequestHandler createAMHeartbeatRequestHandler(
+      Configuration config, ApplicationId appId,
+      AMRMClientRelayer relayer) {
+    return new AMHeartbeatRequestHandler(config, appId, relayer);
+  }
+
   /**
    * Launch a new UAM in the resource manager.
    *
@@ -191,8 +200,6 @@ public class UnmanagedApplicationManager {
         this.applicationId.toString(), UserGroupInformation.getCurrentUser());
     this.rmProxyRelayer.setRMClient(createRMProxy(
         ApplicationMasterProtocol.class, this.conf, this.userUgi, amrmToken));
-
-    this.heartbeatHandler.setAMRMClientRelayer(this.rmProxyRelayer);
     this.heartbeatHandler.setUGI(this.userUgi);
   }
 
@@ -522,11 +529,6 @@ public class UnmanagedApplicationManager {
   }
 
   @VisibleForTesting
-  protected void setHandlerThread(AMHeartbeatRequestHandler thread) {
-    this.heartbeatHandler = thread;
-  }
-
-  @VisibleForTesting
   protected void drainHeartbeatThread() {
     this.heartbeatHandler.drainHeartbeatThread();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 47f85cc..8dca937 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -192,10 +192,15 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
   private boolean shouldWaitForSyncNextAllocate = false;
 
   // For unit test synchronization
-  private static Object syncObj = new Object();
+  private static Object registerSyncObj = new Object();
+  private static Object allocateSyncObj = new Object();
 
-  public static Object getSyncObj() {
-    return syncObj;
+  public static Object getRegisterSyncObj() {
+    return registerSyncObj;
+  }
+
+  public static Object getAllocateSyncObj() {
+    return allocateSyncObj;
   }
 
   public MockResourceManagerFacade(Configuration conf,
@@ -275,14 +280,14 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
     shouldReRegisterNext = false;
 
     // Make sure we wait for certain test cases last in the method
-    synchronized (syncObj) {
-      syncObj.notifyAll();
+    synchronized (registerSyncObj) {
+      registerSyncObj.notifyAll();
       // We reuse the port number to indicate whether the unit test want us to
       // wait here
       if (request.getRpcPort() > 1000) {
         LOG.info("Register call in RM start waiting");
         try {
-          syncObj.wait();
+          registerSyncObj.wait();
           LOG.info("Register call in RM wait finished");
         } catch (InterruptedException e) {
           LOG.info("Register call in RM wait interrupted", e);
@@ -349,13 +354,13 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
     }
 
     // Wait for signal for certain test cases
-    synchronized (syncObj) {
+    synchronized (allocateSyncObj) {
       if (shouldWaitForSyncNextAllocate) {
         shouldWaitForSyncNextAllocate = false;
 
         LOG.info("Allocate call in RM start waiting");
         try {
-          syncObj.wait();
+          allocateSyncObj.wait();
           LOG.info("Allocate call in RM wait finished");
         } catch (InterruptedException e) {
           LOG.info("Allocate call in RM wait interrupted", e);
@@ -416,8 +421,8 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
           }
 
           Assert.assertTrue("ContainerId " + id
-              + " being released is not valid for application: "
-              + conf.get("AMRMTOKEN"), found);
+              + " being released is not valid for application: " + attemptId,
+              found);
 
           ids.remove(id);
           completedList.add(
@@ -427,7 +432,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
     }
 
     LOG.info("Allocating containers: " + containerList.size()
-        + " for application attempt: " + conf.get("AMRMTOKEN"));
+        + " for application attempt: " + attemptId);
 
     // Always issue a new AMRMToken as if RM rolled master key
     Token newAMRMToken = Token.newInstance(new byte[0],
@@ -440,7 +445,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
   }
 
   public void setWaitForSyncNextAllocate(boolean wait) {
-    synchronized (syncObj) {
+    synchronized (allocateSyncObj) {
       shouldWaitForSyncNextAllocate = wait;
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
index b6bb0da..54e7dd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
@@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler;
+import org.apache.hadoop.yarn.server.AMRMClientRelayer;
 import org.apache.hadoop.yarn.server.MockResourceManagerFacade;
 import org.apache.hadoop.yarn.util.AsyncCallback;
 import org.junit.Assert;
@@ -65,7 +67,8 @@ public class TestUnmanagedApplicationManager {
         ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
 
     uam = new TestableUnmanagedApplicationManager(conf,
-        attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true);
+        attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true,
+        "rm");
   }
 
   protected void waitForCallBackCountAndCheckZeroPending(
@@ -121,7 +124,8 @@ public class TestUnmanagedApplicationManager {
 
     MockResourceManagerFacade rmProxy = uam.getRMProxy();
     uam = new TestableUnmanagedApplicationManager(conf,
-        attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true);
+        attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true,
+        "rm");
     uam.setRMProxy(rmProxy);
 
     reAttachUAM(null, attemptId);
@@ -186,7 +190,7 @@ public class TestUnmanagedApplicationManager {
     });
 
     // Sync obj from mock RM
-    Object syncObj = MockResourceManagerFacade.getSyncObj();
+    Object syncObj = MockResourceManagerFacade.getRegisterSyncObj();
 
     // Wait for register call in the thread get into RM and then wake us
     synchronized (syncObj) {
@@ -365,16 +369,24 @@ public class TestUnmanagedApplicationManager {
   /**
    * Testable UnmanagedApplicationManager that talks to a mock RM.
    */
-  public static class TestableUnmanagedApplicationManager
+  public class TestableUnmanagedApplicationManager
       extends UnmanagedApplicationManager {
 
     private MockResourceManagerFacade rmProxy;
 
     public TestableUnmanagedApplicationManager(Configuration conf,
         ApplicationId appId, String queueName, String submitter,
-        String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
+        String appNameSuffix, boolean keepContainersAcrossApplicationAttempts,
+        String rmName) {
       super(conf, appId, queueName, submitter, appNameSuffix,
-          keepContainersAcrossApplicationAttempts, "TEST");
+          keepContainersAcrossApplicationAttempts, rmName);
+    }
+
+    @Override
+    protected AMHeartbeatRequestHandler createAMHeartbeatRequestHandler(
+        Configuration config, ApplicationId appId,
+        AMRMClientRelayer rmProxyRelayer) {
+      return new TestableAMRequestHandlerThread(config, appId, rmProxyRelayer);
     }
 
     @SuppressWarnings("unchecked")
@@ -402,4 +414,31 @@ public class TestUnmanagedApplicationManager {
     }
   }
 
+  /**
+   * Wrap the handler thread so it calls from the same user.
+   */
+  public class TestableAMRequestHandlerThread
+      extends AMHeartbeatRequestHandler {
+    public TestableAMRequestHandlerThread(Configuration conf,
+        ApplicationId applicationId, AMRMClientRelayer rmProxyRelayer) {
+      super(conf, applicationId, rmProxyRelayer);
+    }
+
+    @Override
+    public void run() {
+      try {
+        getUGIWithToken(attemptId)
+            .doAs(new PrivilegedExceptionAction<Object>() {
+              @Override
+              public Object run() {
+                TestableAMRequestHandlerThread.super.run();
+                return null;
+              }
+            });
+      } catch (Exception e) {
+        LOG.error("Exception running TestableAMRequestHandlerThread", e);
+      }
+    }
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index 4267945..c478871 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -167,6 +167,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    */
   private Map<SubClusterId, List<AllocateResponse>> asyncResponseSink;
 
+  private Map<SubClusterId, RegisterApplicationMasterResponse> uamRegistrations;
+
+  // For unit test synchronization
+  private Map<SubClusterId, Future<?>> uamRegisterFutures;
+
   /** Thread pool used for asynchronous operations. */
   private ExecutorService threadpool;
 
@@ -227,6 +232,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   public FederationInterceptor() {
     this.containerIdToSubClusterIdMap = new ConcurrentHashMap<>();
     this.asyncResponseSink = new ConcurrentHashMap<>();
+    this.uamRegistrations = new ConcurrentHashMap<>();
+    this.uamRegisterFutures = new ConcurrentHashMap<>();
     this.threadpool = Executors.newCachedThreadPool();
     this.uamPool = createUnmanagedAMPoolManager(this.threadpool);
     this.secondaryRelayers = new ConcurrentHashMap<>();
@@ -279,8 +286,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
         ApplicationMasterProtocol.class, appOwner), appId,
         this.homeSubClusterId.toString());
 
-    this.homeHeartbeartHandler = createHomeHeartbeartHandler(conf, appId);
-    this.homeHeartbeartHandler.setAMRMClientRelayer(this.homeRMRelayer);
+    this.homeHeartbeartHandler =
+        createHomeHeartbeartHandler(conf, appId, this.homeRMRelayer);
     this.homeHeartbeartHandler.setUGI(appOwner);
     this.homeHeartbeartHandler.setDaemon(true);
     this.homeHeartbeartHandler.start();
@@ -615,10 +622,9 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       /**
        * Send the requests to the all sub-cluster resource managers. All
        * requests are synchronously triggered but sent asynchronously. Later the
-       * responses will be collected and merged. In addition, it also returns
-       * the newly registered UAMs.
+       * responses will be collected and merged.
        */
-      Registrations newRegistrations = sendRequestsToResourceManagers(requests);
+      sendRequestsToResourceManagers(requests);
 
       // Wait for the first async response to arrive
       long startTime = this.clock.getTime();
@@ -646,9 +652,14 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
 
       // Merge the containers and NMTokens from the new registrations into
       // the response
-      if (!isNullOrEmpty(newRegistrations.getSuccessfulRegistrations())) {
-        mergeRegistrationResponses(response,
-            newRegistrations.getSuccessfulRegistrations());
+
+      if (!isNullOrEmpty(this.uamRegistrations)) {
+        Map<SubClusterId, RegisterApplicationMasterResponse> newRegistrations;
+        synchronized (this.uamRegistrations) {
+          newRegistrations = new HashMap<>(this.uamRegistrations);
+          this.uamRegistrations.clear();
+        }
+        mergeRegistrationResponses(response, newRegistrations);
       }
 
       // update the responseId and return the final response to AM
@@ -850,8 +861,9 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
 
   @VisibleForTesting
   protected AMHeartbeatRequestHandler createHomeHeartbeartHandler(
-      Configuration conf, ApplicationId appId) {
-    return new AMHeartbeatRequestHandler(conf, appId);
+      Configuration conf, ApplicationId appId,
+      AMRMClientRelayer rmProxyRelayer) {
+    return new AMHeartbeatRequestHandler(conf, appId, rmProxyRelayer);
   }
 
   /**
@@ -1107,18 +1119,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    *
    * @param requests contains the heart beat requests to send to the resource
    *          manager keyed by the sub-cluster id
-   * @return the registration responses from the newly added sub-cluster
-   *         resource managers
    * @throws YarnException
    * @throws IOException
    */
-  private Registrations sendRequestsToResourceManagers(
+  private void sendRequestsToResourceManagers(
       Map<SubClusterId, AllocateRequest> requests)
       throws YarnException, IOException {
 
-    // Create new UAM instances for the sub-cluster that we have not seen
-    // before
-    Registrations registrations = registerWithNewSubClusters(requests.keySet());
+    // Create new UAM instances for the sub-cluster that we haven't seen before
+    List<SubClusterId> newSubClusters =
+        registerAndAllocateWithNewSubClusters(requests);
 
     // Now that all the registrations are done, send the allocation request
     // to the sub-cluster RMs asynchronously and don't wait for the response.
@@ -1126,6 +1136,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     // response sink, then merged and sent to the application master.
     for (Entry<SubClusterId, AllocateRequest> entry : requests.entrySet()) {
       SubClusterId subClusterId = entry.getKey();
+      if (newSubClusters.contains(subClusterId)) {
+        // For new sub-clusters, we have already sent the request right after
+        // register in the async thread
+        continue;
+      }
 
       if (subClusterId.equals(this.homeSubClusterId)) {
         // Request for the home sub-cluster resource manager
@@ -1133,131 +1148,100 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
             new HeartbeatCallBack(this.homeSubClusterId, false));
       } else {
         if (!this.uamPool.hasUAMId(subClusterId.getId())) {
-          // TODO: This means that the registration for this sub-cluster RM
-          // failed. For now, we ignore the resource requests and continue
-          // but we need to fix this and handle this situation. One way would
-          // be to send the request to another RM by consulting the policy.
-          LOG.warn("Unmanaged AM registration not found for sub-cluster {}",
-              subClusterId);
-          continue;
+          throw new YarnException("UAM not found for " + this.attemptId
+              + " in sub-cluster " + subClusterId);
         }
         this.uamPool.allocateAsync(subClusterId.getId(), entry.getValue(),
             new HeartbeatCallBack(subClusterId, true));
       }
     }
-
-    return registrations;
   }
 
   /**
-   * This method ensures that Unmanaged AMs are created for each of the
-   * specified sub-cluster specified in the input and registers with the
-   * corresponding resource managers.
+   * This method ensures that Unmanaged AMs are created for newly specified
+   * sub-clusters, registers with the corresponding resource managers and send
+   * the first allocate request async.
    */
-  private Registrations registerWithNewSubClusters(
-      Set<SubClusterId> subClusterSet) throws IOException {
-
-    List<SubClusterId> failedRegistrations = new ArrayList<>();
-    Map<SubClusterId, RegisterApplicationMasterResponse>
-        successfulRegistrations = new HashMap<>();
+  private List<SubClusterId> registerAndAllocateWithNewSubClusters(
+      final Map<SubClusterId, AllocateRequest> requests) throws IOException {
 
     // Check to see if there are any new sub-clusters in this request
     // list and create and register Unmanaged AM instance for the new ones
-    List<String> newSubClusters = new ArrayList<>();
-    for (SubClusterId subClusterId : subClusterSet) {
+    List<SubClusterId> newSubClusters = new ArrayList<>();
+    for (SubClusterId subClusterId : requests.keySet()) {
       if (!subClusterId.equals(this.homeSubClusterId)
           && !this.uamPool.hasUAMId(subClusterId.getId())) {
-        newSubClusters.add(subClusterId.getId());
+        newSubClusters.add(subClusterId);
       }
     }
 
-    if (newSubClusters.size() > 0) {
-      final RegisterApplicationMasterRequest registerRequest =
-          this.amRegistrationRequest;
-      final AMRMProxyApplicationContext appContext = getApplicationContext();
-      ExecutorCompletionService<RegisterApplicationMasterResponseInfo>
-          completionService = new ExecutorCompletionService<>(this.threadpool);
+    this.uamRegisterFutures.clear();
+    for (final SubClusterId scId : newSubClusters) {
+      Future<?> future = this.threadpool.submit(new Runnable() {
+        @Override
+        public void run() {
+          String subClusterId = scId.getId();
 
-      for (final String subClusterId : newSubClusters) {
-        completionService
-            .submit(new Callable<RegisterApplicationMasterResponseInfo>() {
-              @Override
-              public RegisterApplicationMasterResponseInfo call()
-                  throws Exception {
-
-                // Create a config loaded with federation on and subclusterId
-                // for each UAM
-                YarnConfiguration config = new YarnConfiguration(getConf());
-                FederationProxyProviderUtil.updateConfForFederation(config,
-                    subClusterId);
-
-                RegisterApplicationMasterResponse uamResponse = null;
-                Token<AMRMTokenIdentifier> token = null;
-                try {
-                  // For appNameSuffix, use subClusterId of the home sub-cluster
-                  token = uamPool.launchUAM(subClusterId, config,
-                      attemptId.getApplicationId(),
-                      amRegistrationResponse.getQueue(), appContext.getUser(),
-                      homeSubClusterId.toString(), true, subClusterId);
-
-                  secondaryRelayers.put(subClusterId,
-                      uamPool.getAMRMClientRelayer(subClusterId));
-
-                  uamResponse = uamPool.registerApplicationMaster(subClusterId,
-                      registerRequest);
-                } catch (Throwable e) {
-                  LOG.error("Failed to register application master: "
-                      + subClusterId + " Application: " + attemptId, e);
-                }
-                return new RegisterApplicationMasterResponseInfo(uamResponse,
-                    SubClusterId.newInstance(subClusterId), token);
-              }
-            });
-      }
+          // Create a config loaded with federation on and subclusterId
+          // for each UAM
+          YarnConfiguration config = new YarnConfiguration(getConf());
+          FederationProxyProviderUtil.updateConfForFederation(config,
+              subClusterId);
 
-      // Wait for other sub-cluster resource managers to return the
-      // response and add it to the Map for returning to the caller
-      for (int i = 0; i < newSubClusters.size(); ++i) {
-        try {
-          Future<RegisterApplicationMasterResponseInfo> future =
-              completionService.take();
-          RegisterApplicationMasterResponseInfo uamResponse = future.get();
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Received register application response from RM: "
-                + uamResponse.getSubClusterId());
+          RegisterApplicationMasterResponse uamResponse = null;
+          Token<AMRMTokenIdentifier> token = null;
+          try {
+            // For appNameSuffix, use subClusterId of the home sub-cluster
+            token = uamPool.launchUAM(subClusterId, config,
+                attemptId.getApplicationId(), amRegistrationResponse.getQueue(),
+                getApplicationContext().getUser(), homeSubClusterId.toString(),
+                true, subClusterId);
+
+            secondaryRelayers.put(subClusterId,
+                uamPool.getAMRMClientRelayer(subClusterId));
+
+            uamResponse = uamPool.registerApplicationMaster(subClusterId,
+                amRegistrationRequest);
+          } catch (Throwable e) {
+            LOG.error("Failed to register application master: " + subClusterId
+                + " Application: " + attemptId, e);
+            // TODO: UAM registration for this sub-cluster RM
+            // failed. For now, we ignore the resource requests and continue
+            // but we need to fix this and handle this situation. One way would
+            // be to send the request to another RM by consulting the policy.
+            return;
           }
+          uamRegistrations.put(scId, uamResponse);
+          LOG.info("Successfully registered unmanaged application master: "
+              + subClusterId + " ApplicationId: " + attemptId);
 
-          if (uamResponse.getResponse() == null) {
-            failedRegistrations.add(uamResponse.getSubClusterId());
-          } else {
-            LOG.info("Successfully registered unmanaged application master: "
-                + uamResponse.getSubClusterId() + " ApplicationId: "
-                + this.attemptId);
-            successfulRegistrations.put(uamResponse.getSubClusterId(),
-                uamResponse.getResponse());
+          try {
+            uamPool.allocateAsync(subClusterId, requests.get(scId),
+                new HeartbeatCallBack(scId, true));
+          } catch (Throwable e) {
+            LOG.error("Failed to allocate async to " + subClusterId
+                + " Application: " + attemptId, e);
+          }
 
-            // Save the UAM token in registry or NMSS
+          // Save the UAM token in registry or NMSS
+          try {
             if (registryClient != null) {
-              registryClient.writeAMRMTokenForUAM(
-                  this.attemptId.getApplicationId(),
-                  uamResponse.getSubClusterId().getId(),
-                  uamResponse.getUamToken());
+              registryClient.writeAMRMTokenForUAM(attemptId.getApplicationId(),
+                  subClusterId, token);
             } else if (getNMStateStore() != null) {
-              getNMStateStore().storeAMRMProxyAppContextEntry(this.attemptId,
-                  NMSS_SECONDARY_SC_PREFIX
-                      + uamResponse.getSubClusterId().getId(),
-                  uamResponse.getUamToken().encodeToUrlString()
-                      .getBytes(STRING_TO_BYTE_FORMAT));
+              getNMStateStore().storeAMRMProxyAppContextEntry(attemptId,
+                  NMSS_SECONDARY_SC_PREFIX + subClusterId,
+                  token.encodeToUrlString().getBytes(STRING_TO_BYTE_FORMAT));
             }
+          } catch (Throwable e) {
+            LOG.error("Failed to persist UAM token from " + subClusterId
+                + " Application: " + attemptId, e);
           }
-        } catch (Exception e) {
-          LOG.warn("Failed to register unmanaged application master: "
-              + " ApplicationId: " + this.attemptId, e);
         }
-      }
+      });
+      this.uamRegisterFutures.put(scId, future);
     }
-
-    return new Registrations(successfulRegistrations, failedRegistrations);
+    return newSubClusters;
   }
 
   /**
@@ -1573,7 +1557,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   }
 
   @VisibleForTesting
-  public int getUnmanagedAMPoolSize() {
+  protected int getUnmanagedAMPoolSize() {
     return this.uamPool.getAllUAMIds().size();
   }
 
@@ -1583,6 +1567,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   }
 
   @VisibleForTesting
+  protected Map<SubClusterId, Future<?>> getUamRegisterFutures() {
+    return this.uamRegisterFutures;
+  }
+
+  @VisibleForTesting
   public Map<SubClusterId, List<AllocateResponse>> getAsyncResponseSink() {
     return this.asyncResponseSink;
   }
@@ -1614,7 +1603,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
         asyncResponseSink.notifyAll();
       }
 
+      // Notify policy of allocate response
+      try {
+        policyInterpreter.notifyOfResponse(subClusterId, response);
+      } catch (YarnException e) {
+        LOG.warn("notifyOfResponse for policy failed for sub-cluster "
+            + subClusterId, e);
+      }
+
       // Save the new AMRMToken for the UAM if present
+      // Do this last because it can be slow...
       if (this.isUAM && response.getAMRMToken() != null) {
         Token<AMRMTokenIdentifier> newToken = ConverterUtils
             .convertFromYarn(response.getAMRMToken(), (Text) null);
@@ -1648,44 +1646,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
           }
         }
       }
-
-      // Notify policy of allocate response
-      try {
-        policyInterpreter.notifyOfResponse(subClusterId, response);
-      } catch (YarnException e) {
-        LOG.warn("notifyOfResponse for policy failed for sub-cluster "
-            + subClusterId, e);
-      }
-    }
-  }
-
-  /**
-   * Private structure for encapsulating SubClusterId and
-   * RegisterApplicationMasterResponse instances.
-   */
-  private static class RegisterApplicationMasterResponseInfo {
-    private RegisterApplicationMasterResponse response;
-    private SubClusterId subClusterId;
-    private Token<AMRMTokenIdentifier> uamToken;
-
-    RegisterApplicationMasterResponseInfo(
-        RegisterApplicationMasterResponse response, SubClusterId subClusterId,
-        Token<AMRMTokenIdentifier> uamToken) {
-      this.response = response;
-      this.subClusterId = subClusterId;
-      this.uamToken = uamToken;
-    }
-
-    public RegisterApplicationMasterResponse getResponse() {
-      return response;
-    }
-
-    public SubClusterId getSubClusterId() {
-      return subClusterId;
-    }
-
-    public Token<AMRMTokenIdentifier> getUamToken() {
-      return uamToken;
     }
   }
 
@@ -1713,33 +1673,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   }
 
   /**
-   * Private structure for encapsulating successful and failed application
-   * master registration responses.
-   */
-  private static class Registrations {
-    private Map<SubClusterId, RegisterApplicationMasterResponse>
-        successfulRegistrations;
-    private List<SubClusterId> failedRegistrations;
-
-    Registrations(
-        Map<SubClusterId, RegisterApplicationMasterResponse>
-            successfulRegistrations,
-        List<SubClusterId> failedRegistrations) {
-      this.successfulRegistrations = successfulRegistrations;
-      this.failedRegistrations = failedRegistrations;
-    }
-
-    public Map<SubClusterId, RegisterApplicationMasterResponse>
-        getSuccessfulRegistrations() {
-      return this.successfulRegistrations;
-    }
-
-    public List<SubClusterId> getFailedRegistrations() {
-      return this.failedRegistrations;
-    }
-  }
-
-  /**
    * Utility method to check if the specified Collection is null or empty.
    *
    * @param c the collection object

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index 407ae83..ec75cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -201,9 +201,6 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
     LOG.info("Number of allocated containers in the original request: "
         + Integer.toString(allocateResponse.getAllocatedContainers().size()));
 
-    // Make sure this request is picked up by all async heartbeat handlers
-    interceptor.drainAllAsyncQueue(false);
-
     // Send max 10 heart beats to receive all the containers. If not, we will
     // fail the test
     int numHeartbeat = 0;
@@ -217,8 +214,10 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
       checkAMRMToken(allocateResponse.getAMRMToken());
       lastResponseId = allocateResponse.getResponseId();
 
-      containers.addAll(allocateResponse.getAllocatedContainers());
+      // Make sure this request is picked up by all async heartbeat handlers
+      interceptor.drainAllAsyncQueue(false);
 
+      containers.addAll(allocateResponse.getAllocatedContainers());
       LOG.info("Number of allocated containers in this request: "
           + Integer.toString(allocateResponse.getAllocatedContainers().size()));
       LOG.info("Total number of allocated containers: "
@@ -258,9 +257,6 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
     LOG.info("Number of containers received in the original request: "
         + Integer.toString(newlyFinished.size()));
 
-    // Make sure this request is picked up by all async heartbeat handlers
-    interceptor.drainAllAsyncQueue(false);
-
     // Send max 10 heart beats to receive all the containers. If not, we will
     // fail the test
     int numHeartbeat = 0;
@@ -273,10 +269,12 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
       checkAMRMToken(allocateResponse.getAMRMToken());
       lastResponseId = allocateResponse.getResponseId();
 
+      // Make sure this request is picked up by all async heartbeat handlers
+      interceptor.drainAllAsyncQueue(false);
+
       newlyFinished = getCompletedContainerIds(
           allocateResponse.getCompletedContainersStatuses());
       containersForReleasedContainerIds.addAll(newlyFinished);
-
       LOG.info("Number of containers received in this request: "
           + Integer.toString(newlyFinished.size()));
       LOG.info("Total number of containers received: "
@@ -438,7 +436,7 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
     ExecutorCompletionService<RegisterApplicationMasterResponse> compSvc =
         new ExecutorCompletionService<>(threadpool);
 
-    Object syncObj = MockResourceManagerFacade.getSyncObj();
+    Object syncObj = MockResourceManagerFacade.getRegisterSyncObj();
 
     // Two register threads
     synchronized (syncObj) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f50f9cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
index 78f6eb0..7c57ab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler;
+import org.apache.hadoop.yarn.server.AMRMClientRelayer;
 import org.apache.hadoop.yarn.server.MockResourceManagerFacade;
 import org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager;
 import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
@@ -69,8 +70,9 @@ public class TestableFederationInterceptor extends FederationInterceptor {
 
   @Override
   protected AMHeartbeatRequestHandler createHomeHeartbeartHandler(
-      Configuration conf, ApplicationId appId) {
-    return new TestableAMRequestHandlerThread(conf, appId);
+      Configuration conf, ApplicationId appId,
+      AMRMClientRelayer rmProxyRelayer) {
+    return new TestableAMRequestHandlerThread(conf, appId, rmProxyRelayer);
   }
 
   @SuppressWarnings("unchecked")
@@ -205,7 +207,8 @@ public class TestableFederationInterceptor extends FederationInterceptor {
         String appNameSuffix, boolean keepContainersAcrossApplicationAttempts,
         String rmId) {
       return new TestableUnmanagedApplicationManager(conf, appId, queueName,
-          submitter, appNameSuffix, keepContainersAcrossApplicationAttempts);
+          submitter, appNameSuffix, keepContainersAcrossApplicationAttempts,
+          rmId);
     }
   }
 
@@ -218,10 +221,17 @@ public class TestableFederationInterceptor extends FederationInterceptor {
 
     public TestableUnmanagedApplicationManager(Configuration conf,
         ApplicationId appId, String queueName, String submitter,
-        String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
+        String appNameSuffix, boolean keepContainersAcrossApplicationAttempts,
+        String rmName) {
       super(conf, appId, queueName, submitter, appNameSuffix,
-          keepContainersAcrossApplicationAttempts, "TEST");
-      setHandlerThread(new TestableAMRequestHandlerThread(conf, appId));
+          keepContainersAcrossApplicationAttempts, rmName);
+    }
+
+    @Override
+    protected AMHeartbeatRequestHandler createAMHeartbeatRequestHandler(
+        Configuration conf, ApplicationId appId,
+        AMRMClientRelayer rmProxyRelayer) {
+      return new TestableAMRequestHandlerThread(conf, appId, rmProxyRelayer);
     }
 
     /**
@@ -244,8 +254,8 @@ public class TestableFederationInterceptor extends FederationInterceptor {
   protected class TestableAMRequestHandlerThread
       extends AMHeartbeatRequestHandler {
     public TestableAMRequestHandlerThread(Configuration conf,
-        ApplicationId applicationId) {
-      super(conf, applicationId);
+        ApplicationId applicationId, AMRMClientRelayer rmProxyRelayer) {
+      super(conf, applicationId, rmProxyRelayer);
     }
 
     @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[16/50] [abbrv] hadoop git commit: YARN-7502. Nodemanager restart docs should describe nodemanager supervised property. Contributed by Suma Shivaprasad.

Posted by jh...@apache.org.
YARN-7502. Nodemanager restart docs should describe nodemanager supervised property. Contributed by Suma Shivaprasad.

(cherry picked from commit e50334513c781a10368e7353bec6674f89eb0a7b)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c39681b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c39681b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c39681b

Branch: refs/heads/branch-2-jhung-test
Commit: 0c39681b2cc24723171c34676760a86ffefeda07
Parents: 03ff0f1
Author: Sunil G <su...@apache.org>
Authored: Mon Oct 22 14:15:51 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop-yarn-site/src/site/markdown/NodeManager.md     | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c39681b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
index 12201b9..18aeff4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeManager.md
@@ -77,13 +77,19 @@ Step 2.  Configure a path to the local file-system directory where the NodeManag
 |:---- |:---- |
 | `yarn.nodemanager.recovery.dir` | The local filesystem directory in which the node manager will store state when recovery is enabled. The default value is set to `$hadoop.tmp.dir/yarn-nm-recovery`. |
 
-Step 3.  Configure a valid RPC address for the NodeManager.
+Step 3: Enable NM supervision under recovery to prevent running containers from getting cleaned up when NM exits.
+
+| Property | Description |
+|:---- |:---- |
+| `yarn.nodemanager.recovery.supervised` | If enabled, NodeManager running will not try to cleanup containers as it exits with the assumption it will be immediately be restarted and recover containers The default value is set to 'false'. |
+
+Step 4.  Configure a valid RPC address for the NodeManager.
 
 | Property | Description |
 |:---- |:---- |
 | `yarn.nodemanager.address` | Ephemeral ports (port 0, which is default) cannot be used for the NodeManager's RPC server specified via yarn.nodemanager.address as it can make NM use different ports before and after a restart. This will break any previously running clients that were communicating with the NM before restart. Explicitly setting yarn.nodemanager.address to an address with specific port number (for e.g 0.0.0.0:45454) is a precondition for enabling NM restart. |
 
-Step 4.  Auxiliary services.
+Step 5.  Auxiliary services.
 
   * NodeManagers in a YARN cluster can be configured to run auxiliary services. For a completely functional NM restart, YARN relies on any auxiliary service configured to also support recovery. This usually includes (1) avoiding usage of ephemeral ports so that previously running clients (in this case, usually containers) are not disrupted after restart and (2) having the auxiliary service itself support recoverability by reloading any previous state when NodeManager restarts and reinitializes the auxiliary service.
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[42/50] [abbrv] hadoop git commit: YARN-8233. NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null. Contributed by Tao Yang.

Posted by jh...@apache.org.
YARN-8233. NPE in CapacityScheduler#tryCommit when handling allocate/reserve proposal whose allocatedOrReservedContainer is null. Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/181e1388
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/181e1388
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/181e1388

Branch: refs/heads/branch-2-jhung-test
Commit: 181e13880c8bffb18e16cf1203caa1da768189fa
Parents: b86c651
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sat Nov 10 21:44:46 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/CapacityScheduler.java   | 72 ++++++++++-----
 .../TestCapacitySchedulerAsyncScheduling.java   | 94 ++++++++++++++++++++
 2 files changed, 143 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/181e1388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 207e059..e70e7cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2379,7 +2379,11 @@ public class CapacityScheduler extends
         .getContainersToKill().isEmpty()) {
       list = new ArrayList<>();
       for (RMContainer rmContainer : csAssignment.getContainersToKill()) {
-        list.add(getSchedulerContainer(rmContainer, false));
+        SchedulerContainer schedulerContainer =
+            getSchedulerContainer(rmContainer, false);
+        if (schedulerContainer != null) {
+          list.add(schedulerContainer);
+        }
       }
     }
 
@@ -2387,10 +2391,16 @@ public class CapacityScheduler extends
       if (null == list) {
         list = new ArrayList<>();
       }
-      list.add(
-          getSchedulerContainer(csAssignment.getExcessReservation(), false));
+      SchedulerContainer schedulerContainer =
+          getSchedulerContainer(csAssignment.getExcessReservation(), false);
+      if (schedulerContainer != null) {
+        list.add(schedulerContainer);
+      }
     }
 
+    if (list != null && list.isEmpty()) {
+      list = null;
+    }
     return list;
   }
 
@@ -2430,16 +2440,27 @@ public class CapacityScheduler extends
           csAssignment.getAssignmentInformation().getAllocationDetails();
       if (!allocations.isEmpty()) {
         RMContainer rmContainer = allocations.get(0).rmContainer;
-        allocated = new ContainerAllocationProposal<>(
-            getSchedulerContainer(rmContainer, true),
-            getSchedulerContainersToRelease(csAssignment),
-            getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
-                false), csAssignment.getType(),
-            csAssignment.getRequestLocalityType(),
-            csAssignment.getSchedulingMode() != null ?
-                csAssignment.getSchedulingMode() :
-                SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
-            csAssignment.getResource());
+        SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+            schedulerContainer = getSchedulerContainer(rmContainer, true);
+        if (schedulerContainer == null) {
+          allocated = null;
+          // Decrease unconfirmed resource if app is alive
+          FiCaSchedulerApp app = getApplicationAttempt(
+              rmContainer.getApplicationAttemptId());
+          if (app != null) {
+            app.decUnconfirmedRes(rmContainer.getAllocatedResource());
+          }
+        } else {
+          allocated = new ContainerAllocationProposal<>(schedulerContainer,
+              getSchedulerContainersToRelease(csAssignment),
+              getSchedulerContainer(
+                  csAssignment.getFulfilledReservedContainer(), false),
+              csAssignment.getType(), csAssignment.getRequestLocalityType(),
+              csAssignment.getSchedulingMode() != null ?
+                  csAssignment.getSchedulingMode() :
+                  SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
+              csAssignment.getResource());
+        }
       }
 
       // Reserved something
@@ -2447,16 +2468,21 @@ public class CapacityScheduler extends
           csAssignment.getAssignmentInformation().getReservationDetails();
       if (!reservation.isEmpty()) {
         RMContainer rmContainer = reservation.get(0).rmContainer;
-        reserved = new ContainerAllocationProposal<>(
-            getSchedulerContainer(rmContainer, false),
-            getSchedulerContainersToRelease(csAssignment),
-            getSchedulerContainer(csAssignment.getFulfilledReservedContainer(),
-                false), csAssignment.getType(),
-            csAssignment.getRequestLocalityType(),
-            csAssignment.getSchedulingMode() != null ?
-                csAssignment.getSchedulingMode() :
-                SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
-            csAssignment.getResource());
+        SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
+            schedulerContainer = getSchedulerContainer(rmContainer, false);
+        if (schedulerContainer == null) {
+          reserved = null;
+        } else {
+          reserved = new ContainerAllocationProposal<>(schedulerContainer,
+              getSchedulerContainersToRelease(csAssignment),
+              getSchedulerContainer(
+                  csAssignment.getFulfilledReservedContainer(), false),
+              csAssignment.getType(), csAssignment.getRequestLocalityType(),
+              csAssignment.getSchedulingMode() != null ?
+                  csAssignment.getSchedulingMode() :
+                  SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
+              csAssignment.getResource());
+        }
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/181e1388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 7e092e8..28c81b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -18,12 +18,14 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
+import com.google.common.collect.ImmutableList;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -40,6 +42,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -52,6 +57,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
 import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
@@ -485,6 +492,93 @@ public class TestCapacitySchedulerAsyncScheduling {
     rm.stop();
   }
 
+  @Test(timeout = 30000)
+  public void testReturnNullWhenGetSchedulerContainer() throws Exception {
+    // disable async-scheduling for simulating complex scenario
+    Configuration disableAsyncConf = new Configuration(conf);
+    disableAsyncConf.setBoolean(
+        CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+    // init RM & NMs
+    final MockRM rm = new MockRM(disableAsyncConf);
+    rm.start();
+    final MockNM nm1 = rm.registerNode("192.168.0.1:1234", 8 * GB);
+    final MockNM nm2 = rm.registerNode("192.168.0.2:2234", 8 * GB);
+    rm.drainEvents();
+    final CapacityScheduler cs =
+        (CapacityScheduler) rm.getRMContext().getScheduler();
+    SchedulerNode sn1 = cs.getSchedulerNode(nm1.getNodeId());
+    final RMNode rmNode1 = cs.getNode(nm1.getNodeId()).getRMNode();
+    SchedulerNode sn2 = cs.getSchedulerNode(nm2.getNodeId());
+
+    // launch app1-am on nm1
+    RMApp app1 = rm.submitApp(1 * GB, "app1", "user", null, false, "default",
+        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true);
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+    // app2 asks 1 * 1G container
+    am1.allocate(ImmutableList.of(ResourceRequest
+        .newInstance(Priority.newInstance(0), "*",
+            Resources.createResource(1 * GB), 1)), null);
+    final RMContainer amContainer = cs.getRMContainer(
+        ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
+
+    // spy CapacityScheduler
+    final CapacityScheduler spyCs = Mockito.spy(cs);
+    // hook CapacityScheduler#submitResourceCommitRequest
+    final List<CSAssignment> assignmentSnapshots = new ArrayList<>();
+    Mockito.doAnswer(new Answer<Object>() {
+      public Boolean answer(InvocationOnMock invocation) throws Exception {
+        CSAssignment assignment = (CSAssignment) invocation.getArguments()[1];
+        if (cs.getNode(nm1.getNodeId()) != null) {
+          // decommission nm1 for first allocation on nm1
+          cs.getRMContext().getDispatcher().getEventHandler().handle(
+              new RMNodeEvent(nm1.getNodeId(), RMNodeEventType.DECOMMISSION));
+          rm.drainEvents();
+          Assert.assertEquals(NodeState.DECOMMISSIONED, rmNode1.getState());
+          Assert.assertNull(cs.getNode(nm1.getNodeId()));
+          assignmentSnapshots.add(assignment);
+        } else {
+          // add am container on nm1 to containersToKill
+          // for second allocation on nm2
+          assignment.setContainersToKill(ImmutableList.of(amContainer));
+        }
+        // check no NPE in actual submit, before YARN-8233 will throw NPE
+        cs.submitResourceCommitRequest((Resource) invocation.getArguments()[0],
+            assignment);
+        return false;
+      }
+    }).when(spyCs).submitResourceCommitRequest(Mockito.any(Resource.class),
+        Mockito.any(CSAssignment.class));
+
+    // allocation on nm1, test return null when get scheduler container
+    PlacementSet<FiCaSchedulerNode> candidateNodeSet =
+        new SimplePlacementSet(sn1);
+    spyCs.allocateContainersToNode(candidateNodeSet, false);
+    // make sure unconfirmed resource is decreased correctly
+    Assert.assertTrue(spyCs.getApplicationAttempt(am1.getApplicationAttemptId())
+        .hasPendingResourceRequest(
+            rm.getResourceScheduler().getResourceCalculator(),
+            RMNodeLabelsManager.NO_LABEL,
+            rm.getResourceScheduler().getClusterResource(),
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+
+    // allocation on nm2,
+    // test return null when get scheduler container to release
+    candidateNodeSet =
+        new SimplePlacementSet(sn2);
+    spyCs.allocateContainersToNode(candidateNodeSet, false);
+    // make sure unconfirmed resource is decreased correctly
+    Assert.assertTrue(spyCs.getApplicationAttempt(am1.getApplicationAttemptId())
+        .hasPendingResourceRequest(
+            rm.getResourceScheduler().getResourceCalculator(),
+            RMNodeLabelsManager.NO_LABEL,
+            rm.getResourceScheduler().getClusterResource(),
+            SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
+
+    rm.stop();
+  }
+
   private void allocateAndLaunchContainers(MockAM am, MockNM nm, MockRM rm,
       int nContainer, Resource resource, int priority, int startContainerId)
       throws Exception {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[19/50] [abbrv] hadoop git commit: YARN-8826. Fix lingering timeline collector after serviceStop in TimelineCollectorManager. Contributed by Prabha Manepalli.

Posted by jh...@apache.org.
YARN-8826. Fix lingering timeline collector after serviceStop in TimelineCollectorManager. Contributed by Prabha Manepalli.

(cherry picked from commit 0b62983c5a9361eb832784f134f140f9926c9ec6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b32ef142
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b32ef142
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b32ef142

Branch: refs/heads/branch-2-jhung-test
Commit: b32ef1429f384af5c6f278c934d511c8875f354d
Parents: 0c39681
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Oct 23 12:58:41 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../server/timelineservice/collector/TimelineCollectorManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b32ef142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index c64a3ad..9727850 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -221,7 +221,7 @@ public class TimelineCollectorManager extends CompositeService {
 
   @Override
   protected void serviceStop() throws Exception {
-    if (collectors != null && collectors.size() > 1) {
+    if (collectors != null && collectors.size() > 0) {
       synchronized (collectors) {
         for (TimelineCollector c : collectors.values()) {
           c.serviceStop();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[50/50] [abbrv] hadoop git commit: Revert "add --no-cache to try to repro branch-2 build failures"

Posted by jh...@apache.org.
Revert "add --no-cache to try to repro branch-2 build failures"

This reverts commit 9bd32e3322874b40d90378f315f927a6bcc8d140.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c02975e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c02975e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c02975e

Branch: refs/heads/branch-2-jhung-test
Commit: 5c02975e9a04fdd8e39d33274400c9c83043492a
Parents: 8aa4c7d
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Dec 14 16:00:06 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 start-build-env.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c02975e/start-build-env.sh
----------------------------------------------------------------------
diff --git a/start-build-env.sh b/start-build-env.sh
index f40a5d1..18e3a8c 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -19,7 +19,7 @@ set -e               # exit on error
 
 cd "$(dirname "$0")" # connect to root
 
-docker build --no-cache -t hadoop-build dev-support/docker
+docker build -t hadoop-build dev-support/docker
 
 if [ "$(uname -s)" == "Linux" ]; then
   USER_NAME=${SUDO_USER:=$USER}
@@ -31,7 +31,7 @@ else # boot2docker uid and gid
   GROUP_ID=50
 fi
 
-docker build --no-cache -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
+docker build -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
 FROM hadoop-build
 RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
 RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER_NAME}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[31/50] [abbrv] hadoop git commit: HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak.

Posted by jh...@apache.org.
HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46f0f4c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46f0f4c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46f0f4c1

Branch: refs/heads/branch-2-jhung-test
Commit: 46f0f4c1d1ad91d02beaa63659345238174266e4
Parents: a121955
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Dec 3 13:23:10 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop/security/LdapGroupsMapping.java      | 163 +++++++++++++------
 .../src/main/resources/core-default.xml         |  28 +++-
 .../src/site/markdown/GroupsMapping.md          |  54 +++++-
 .../hadoop/security/TestLdapGroupsMapping.java  |  80 ++++-----
 .../security/TestLdapGroupsMappingBase.java     |  76 ++++++++-
 .../TestLdapGroupsMappingWithFailover.java      | 142 ++++++++++++++++
 .../TestLdapGroupsMappingWithOneQuery.java      |  16 +-
 .../TestLdapGroupsMappingWithPosixGroup.java    |  10 +-
 8 files changed, 450 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 6beaa9e..83eb5ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Hashtable;
+import java.util.Iterator;
 import java.util.List;
 import java.util.HashSet;
 import java.util.Collection;
@@ -40,7 +41,10 @@ import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 import javax.naming.ldap.LdapName;
 import javax.naming.ldap.Rdn;
+import javax.naming.spi.InitialContextFactory;
 
+import com.google.common.collect.Iterators;
+import com.sun.jndi.ldap.LdapCtxFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
@@ -83,7 +87,7 @@ public class LdapGroupsMapping
   public static final String LDAP_CONFIG_PREFIX = "hadoop.security.group.mapping.ldap";
 
   /*
-   * URL of the LDAP server
+   * URL of the LDAP server(s)
    */
   public static final String LDAP_URL_KEY = LDAP_CONFIG_PREFIX + ".url";
   public static final String LDAP_URL_DEFAULT = "";
@@ -232,6 +236,20 @@ public class LdapGroupsMapping
       LDAP_CONFIG_PREFIX + ".read.timeout.ms";
   public static final int READ_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds
 
+  public static final String LDAP_NUM_ATTEMPTS_KEY =
+      LDAP_CONFIG_PREFIX + ".num.attempts";
+  public static final int LDAP_NUM_ATTEMPTS_DEFAULT = 3;
+
+  public static final String LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY =
+      LDAP_CONFIG_PREFIX + ".num.attempts.before.failover";
+  public static final int LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT =
+      LDAP_NUM_ATTEMPTS_DEFAULT;
+
+  public static final String LDAP_CTX_FACTORY_CLASS_KEY =
+      LDAP_CONFIG_PREFIX + ".ctx.factory.class";
+  public static final Class<? extends LdapCtxFactory>
+      LDAP_CTX_FACTORY_CLASS_DEFAULT = LdapCtxFactory.class;
+
   private static final Logger LOG =
       LoggerFactory.getLogger(LdapGroupsMapping.class);
 
@@ -242,8 +260,10 @@ public class LdapGroupsMapping
 
   private DirContext ctx;
   private Configuration conf;
-  
-  private String ldapUrl;
+
+  private Iterator<String> ldapUrls;
+  private String currentLdapUrl;
+
   private boolean useSsl;
   private String keystore;
   private String keystorePass;
@@ -258,14 +278,15 @@ public class LdapGroupsMapping
   private String memberOfAttr;
   private String groupMemberAttr;
   private String groupNameAttr;
-  private int    groupHierarchyLevels;
+  private int groupHierarchyLevels;
   private String posixUidAttr;
   private String posixGidAttr;
   private boolean isPosix;
   private boolean useOneQuery;
+  private int numAttempts;
+  private int numAttemptsBeforeFailover;
+  private Class<? extends InitialContextFactory> ldapCxtFactoryClass;
 
-  public static final int RECONNECT_RETRY_COUNT = 3;
-  
   /**
    * Returns list of groups for a user.
    * 
@@ -279,20 +300,31 @@ public class LdapGroupsMapping
   @Override
   public synchronized List<String> getGroups(String user) {
     /*
-     * Normal garbage collection takes care of removing Context instances when they are no longer in use. 
-     * Connections used by Context instances being garbage collected will be closed automatically.
-     * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. 
+     * Normal garbage collection takes care of removing Context instances when
+     * they are no longer in use. Connections used by Context instances being
+     * garbage collected will be closed automatically. So in case connection is
+     * closed and gets CommunicationException, retry some times with new new
+     * DirContext/connection.
      */
-    for(int retry = 0; retry < RECONNECT_RETRY_COUNT; retry++) {
+
+    // Tracks the number of attempts made using the same LDAP server
+    int atemptsBeforeFailover = 1;
+
+    for (int attempt = 1; attempt <= numAttempts; attempt++,
+        atemptsBeforeFailover++) {
       try {
         return doGetGroups(user, groupHierarchyLevels);
       } catch (NamingException e) {
-        LOG.warn("Failed to get groups for user " + user + " (retry=" + retry
-            + ") by " + e);
+        LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
+            "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
         LOG.trace("TRACE", e);
+
+        if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
+          atemptsBeforeFailover = 0;
+        }
       }
 
-      //reset ctx so that new DirContext can be created with new connection
+      // Reset ctx so that new DirContext can be created with new connection
       this.ctx = null;
     }
     
@@ -378,10 +410,10 @@ public class LdapGroupsMapping
   private List<String> lookupGroup(SearchResult result, DirContext c,
       int goUpHierarchy)
       throws NamingException {
-    List<String> groups = new ArrayList<String>();
-    Set<String> groupDNs = new HashSet<String>();
+    List<String> groups = new ArrayList<>();
+    Set<String> groupDNs = new HashSet<>();
 
-    NamingEnumeration<SearchResult> groupResults = null;
+    NamingEnumeration<SearchResult> groupResults;
     // perform the second LDAP query
     if (isPosix) {
       groupResults = lookupPosixGroup(result, c);
@@ -402,10 +434,10 @@ public class LdapGroupsMapping
       }
       if (goUpHierarchy > 0 && !isPosix) {
         // convert groups to a set to ensure uniqueness
-        Set<String> groupset = new HashSet<String>(groups);
+        Set<String> groupset = new HashSet<>(groups);
         goUpGroupHierarchy(groupDNs, goUpHierarchy, groupset);
         // convert set back to list for compatibility
-        groups = new ArrayList<String>(groupset);
+        groups = new ArrayList<>(groupset);
       }
     }
     return groups;
@@ -433,11 +465,9 @@ public class LdapGroupsMapping
         userSearchFilter, new Object[]{user}, SEARCH_CONTROLS);
     // return empty list if the user can not be found.
     if (!results.hasMoreElements()) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("doGetGroups(" + user + ") returned no groups because the " +
-            "user is not found.");
-      }
-      return new ArrayList<String>();
+      LOG.debug("doGetGroups({}) returned no groups because the " +
+          "user is not found.", user);
+      return new ArrayList<>();
     }
     SearchResult result = results.nextElement();
 
@@ -455,7 +485,7 @@ public class LdapGroupsMapping
               memberOfAttr + "' attribute." +
               "Returned user object: " + result.toString());
         }
-        groups = new ArrayList<String>();
+        groups = new ArrayList<>();
         NamingEnumeration groupEnumeration = groupDNAttr.getAll();
         while (groupEnumeration.hasMore()) {
           String groupDN = groupEnumeration.next().toString();
@@ -470,9 +500,7 @@ public class LdapGroupsMapping
     if (groups == null || groups.isEmpty() || goUpHierarchy > 0) {
       groups = lookupGroup(result, c, goUpHierarchy);
     }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("doGetGroups(" + user + ") returned " + groups);
-    }
+    LOG.debug("doGetGroups({}) returned {}", user, groups);
     return groups;
   }
 
@@ -480,7 +508,7 @@ public class LdapGroupsMapping
   */
   void getGroupNames(SearchResult groupResult, Collection<String> groups,
                      Collection<String> groupDNs, boolean doGetDNs)
-                     throws NamingException  {
+                     throws NamingException {
     Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
     if (groupName == null) {
       throw new NamingException("The group object does not have " +
@@ -517,7 +545,7 @@ public class LdapGroupsMapping
       return;
     }
     DirContext context = getDirContext();
-    Set<String> nextLevelGroups = new HashSet<String>();
+    Set<String> nextLevelGroups = new HashSet<>();
     StringBuilder filter = new StringBuilder();
     filter.append("(&").append(groupSearchFilter).append("(|");
     for (String dn : groupDNs) {
@@ -537,13 +565,32 @@ public class LdapGroupsMapping
     goUpGroupHierarchy(nextLevelGroups, goUpHierarchy - 1, groups);
   }
 
-  DirContext getDirContext() throws NamingException {
+  /**
+   * Check whether we should fail over to the next LDAP server.
+   * @param attemptsMadeWithSameLdap current number of attempts made
+   *                                 with using same LDAP instance
+   * @param maxAttemptsBeforeFailover maximum number of attempts
+   *                                  before failing over
+   * @return true if we should fail over to the next LDAP server
+   */
+  protected boolean failover(
+      int attemptsMadeWithSameLdap, int maxAttemptsBeforeFailover) {
+    if (attemptsMadeWithSameLdap >= maxAttemptsBeforeFailover) {
+      String previousLdapUrl = currentLdapUrl;
+      currentLdapUrl = ldapUrls.next();
+      LOG.info("Reached {} attempts on {}, failing over to {}",
+          attemptsMadeWithSameLdap, previousLdapUrl, currentLdapUrl);
+      return true;
+    }
+    return false;
+  }
+
+  private DirContext getDirContext() throws NamingException {
     if (ctx == null) {
       // Set up the initial environment for LDAP connectivity
-      Hashtable<String, String> env = new Hashtable<String, String>();
-      env.put(Context.INITIAL_CONTEXT_FACTORY,
-          com.sun.jndi.ldap.LdapCtxFactory.class.getName());
-      env.put(Context.PROVIDER_URL, ldapUrl);
+      Hashtable<String, String> env = new Hashtable<>();
+      env.put(Context.INITIAL_CONTEXT_FACTORY, ldapCxtFactoryClass.getName());
+      env.put(Context.PROVIDER_URL, currentLdapUrl);
       env.put(Context.SECURITY_AUTHENTICATION, "simple");
 
       // Set up SSL security, if necessary
@@ -581,7 +628,7 @@ public class LdapGroupsMapping
    * Caches groups, no need to do that for this provider
    */
   @Override
-  public void cacheGroupsRefresh() throws IOException {
+  public void cacheGroupsRefresh() {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -591,7 +638,7 @@ public class LdapGroupsMapping
    * @param groups unused
    */
   @Override
-  public void cacheGroupsAdd(List<String> groups) throws IOException {
+  public void cacheGroupsAdd(List<String> groups) {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -602,10 +649,12 @@ public class LdapGroupsMapping
 
   @Override
   public synchronized void setConf(Configuration conf) {
-    ldapUrl = conf.get(LDAP_URL_KEY, LDAP_URL_DEFAULT);
-    if (ldapUrl == null || ldapUrl.isEmpty()) {
-      throw new RuntimeException("LDAP URL is not configured");
+    String[] urls = conf.getStrings(LDAP_URL_KEY, LDAP_URL_DEFAULT);
+    if (urls == null || urls.length == 0) {
+      throw new RuntimeException("LDAP URL(s) are not configured");
     }
+    ldapUrls = Iterators.cycle(urls);
+    currentLdapUrl = ldapUrls.next();
 
     useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
     if (useSsl) {
@@ -621,17 +670,13 @@ public class LdapGroupsMapping
     
     String baseDN = conf.getTrimmed(BASE_DN_KEY, BASE_DN_DEFAULT);
 
-    //User search base which defaults to base dn.
+    // User search base which defaults to base dn.
     userbaseDN = conf.getTrimmed(USER_BASE_DN_KEY, baseDN);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Usersearch baseDN: " + userbaseDN);
-    }
+    LOG.debug("Usersearch baseDN: {}", userbaseDN);
 
-    //Group search base which defaults to base dn.
+    // Group search base which defaults to base dn.
     groupbaseDN = conf.getTrimmed(GROUP_BASE_DN_KEY, baseDN);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Groupsearch baseDN: " + userbaseDN);
-    }
+    LOG.debug("Groupsearch baseDN: {}", groupbaseDN);
 
     groupSearchFilter =
         conf.get(GROUP_SEARCH_FILTER_KEY, GROUP_SEARCH_FILTER_DEFAULT);
@@ -655,7 +700,8 @@ public class LdapGroupsMapping
     posixGidAttr =
         conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
 
-    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
+    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
+        DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
     SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
     // Limit the attributes returned to only those required to speed up the search.
     // See HADOOP-10626 and HADOOP-12001 for more details.
@@ -669,9 +715,26 @@ public class LdapGroupsMapping
     }
     SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
 
+    ldapCxtFactoryClass = conf.getClass(LDAP_CTX_FACTORY_CLASS_KEY,
+        LDAP_CTX_FACTORY_CLASS_DEFAULT, InitialContextFactory.class);
+
+    this.numAttempts = conf.getInt(LDAP_NUM_ATTEMPTS_KEY,
+        LDAP_NUM_ATTEMPTS_DEFAULT);
+    this.numAttemptsBeforeFailover = conf.getInt(
+        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT);
+
     this.conf = conf;
   }
 
+  /**
+   * Get URLs of configured LDAP servers.
+   * @return URLs of LDAP servers being used.
+   */
+  public Iterator<String> getLdapUrls() {
+    return ldapUrls;
+  }
+
   private void loadSslConf(Configuration sslConf) {
     keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
     keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY,
@@ -721,8 +784,8 @@ public class LdapGroupsMapping
         password = new String(passchars);
       }
     } catch (IOException ioe) {
-      LOG.warn("Exception while trying to get password for alias " + alias
-              + ": ", ioe);
+      LOG.warn("Exception while trying to get password for alias {}:",
+          alias, ioe);
     }
     return password;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e262b33..e5ce728 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -234,11 +234,35 @@
 </property>
 
 <property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+  <value>3</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations.
+    If this limit is exceeded, LdapGroupsMapping will return an empty
+    group list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+  <value>3</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations
+    using a single LDAP instance. If multiple LDAP servers are configured
+    and this number of failed operations is reached, we will switch to the
+    next LDAP server. The configuration for the overall number of attempts
+    will still be respected, failover will thus be performed only if this
+    property is less than hadoop.security.group.mapping.ldap.num.attempts.
+  </description>
+</property>
+
+<property>
   <name>hadoop.security.group.mapping.ldap.url</name>
   <value></value>
   <description>
-    The URL of the LDAP server to use for resolving user groups when using
-    the LdapGroupsMapping user to group mapping.
+    The URL of the LDAP server(s) to use for resolving user groups when using
+    the LdapGroupsMapping user to group mapping. Supports configuring multiple
+    LDAP servers via a comma-separated list.
   </description>
 </property>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index c6af930..b957821 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -73,7 +73,7 @@ The cache invalidation is configurable via `hadoop.security.groups.negative-cach
 LDAP Groups Mapping
 --------
 This provider supports LDAP with simple password authentication using JNDI API.
-`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server for resolving user groups.
+`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server(s) for resolving user groups. It supports configuring multiple LDAP servers via a comma-separated list.
 
 `hadoop.security.group.mapping.ldap.base` configures the search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.
 Get groups for a given username first looks up the user and then looks up the groups for the user result. If the directory setup has different user and group search bases, use `hadoop.security.group.mapping.ldap.userbase` and `hadoop.security.group.mapping.ldap.groupbase` configs.
@@ -115,6 +115,58 @@ To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memb
 If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`.
 Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`.
 
+### Configuring retries and multiple LDAP servers with failover ###
+If there are issues encountered when retrieving information from LDAP servers, the request will be retried. To configure the number of retries, use the following configuration:
+
+```<property>
+     <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+     <value>3</value>
+     <description>
+       This property is the number of attempts to be made for LDAP operations.
+       If this limit is exceeded, LdapGroupsMapping will return an empty
+       group list.
+     </description>
+    </property>
+```
+
+LDAP Groups Mapping also supports configuring multiple LDAP servers and failover if a particular instance is not available or is misbehaving.
+The following configuration shows configuring 3 LDAP servers. Additionally, 2 attempts will be made for each server before failing over to the next one, with 6 attempts overall before failing.
+
+```
+<property>
+  <name>hadoop.security.group.mapping.ldap.url</name>
+  <value>ldap://server1,ldap://server2,ldap://server3</value>
+  <description>
+    The URL of the LDAP server(s) to use for resolving user groups when using
+    the LdapGroupsMapping user to group mapping. Supports configuring multiple
+    LDAP servers via a comma-separated list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+  <value>6</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations.
+    If this limit is exceeded, LdapGroupsMapping will return an empty
+    group list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+  <value>2</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations
+    using a single LDAP instance. If multiple LDAP servers are configured
+    and this number of failed operations is reached, we will switch to the
+    next LDAP server. The configuration for the overall number of attempts
+    will still be respected, failover will thus be performed only if this
+    property is less than hadoop.security.group.mapping.ldap.num.attempts.
+  </description>
+</property>
+```
+
 Composite Groups Mapping
 --------
 `CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index d9f0c2a..a361d04 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -18,11 +18,12 @@
 package org.apache.hadoop.security;
 
 import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
 import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
@@ -82,14 +83,16 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   private final String userDN = "CN=some_user,DC=test,DC=com";
 
+  private static final String TEST_LDAP_URL = "ldap://test";
+
   @Before
-  public void setupMocks() throws NamingException {
+  public void setupMocks() {
     when(getUserSearchResult().getNameInNamespace()).
         thenReturn(userDN);
   }
   
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -100,9 +103,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDifferentBaseDNs() throws Exception {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     String userBaseDN = "ou=Users,dc=xxx,dc=com ";
     String groupBaseDN = " ou=Groups,dc=xxx,dc=com";
     conf.set(LdapGroupsMapping.USER_BASE_DN_KEY, userBaseDN);
@@ -113,9 +114,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDefaultBaseDN() throws Exception {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     String baseDN = " dc=xxx,dc=com ";
     conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN);
     doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim());
@@ -159,7 +158,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithHierarchy() throws IOException, NamingException {
+  public void testGetGroupsWithHierarchy() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     // The parent search is run once for each level, and is a different search
@@ -175,7 +174,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
+  public void testGetGroupsWithConnectionClosed() throws NamingException {
     // The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
     // then after reconnected return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -190,21 +189,20 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithLdapDown() throws IOException, NamingException {
+  public void testGetGroupsWithLdapDown() throws NamingException {
     // This mocks the case where Ldap server is down, and always throws CommunicationException 
     when(getContext().search(anyString(), anyString(), any(Object[].class),
         any(SearchControls.class)))
         .thenThrow(new CommunicationException("Connection is closed"));
     
     // Ldap server is down, no groups should be retrieved
-    doTestGetGroups(Arrays.asList(new String[] {}), 
-        LdapGroupsMapping.RECONNECT_RETRY_COUNT);
+    doTestGetGroups(Arrays.asList(new String[] {}), 4);
   }
   
-  private void doTestGetGroups(List<String> expectedGroups, int searchTimes) throws IOException, NamingException {  
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+  private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
+      throws NamingException {
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, searchTimes);
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(conf);
@@ -216,17 +214,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     
     // We should have searched for a user, and then two groups
     verify(getContext(), times(searchTimes)).search(anyString(),
-                                         anyString(),
-                                         any(Object[].class),
-                                         any(SearchControls.class));
+        anyString(), any(Object[].class), any(SearchControls.class));
   }
 
   private void doTestGetGroupsWithParent(List<String> expectedGroups,
-      int searchTimesGroup, int searchTimesParentGroup)
-          throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      int searchTimesGroup, int searchTimesParentGroup) throws NamingException {
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     // Set the config to get parents 1 level up
     conf.setInt(LdapGroupsMapping.GROUP_HIERARCHY_LEVELS_KEY, 1);
 
@@ -237,18 +230,14 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     List<String> groups = groupsMapping.getGroups("some_user");
 
     // compare lists, ignoring the order
-    Assert.assertEquals(new HashSet<String>(expectedGroups),
-        new HashSet<String>(groups));
+    Assert.assertEquals(new HashSet<>(expectedGroups), new HashSet<>(groups));
 
     // We should have searched for a user, and group
     verify(getContext(), times(searchTimesGroup)).search(anyString(),
-                                         anyString(),
-                                         any(Object[].class),
-                                         any(SearchControls.class));
+        anyString(), any(Object[].class), any(SearchControls.class));
     // One groups search for the parent group should have been done
-    verify(getContext(), times(searchTimesParentGroup)).search(anyString(),
-                                         anyString(),
-                                         any(SearchControls.class));
+    verify(getContext(), times(searchTimesParentGroup)).search(
+        anyString(), anyString(), any(SearchControls.class));
   }
 
   @Test
@@ -268,7 +257,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   @Test
   public void testConfGetPassword() throws Exception {
     File testDir = GenericTestUtils.getTestDir();
-    Configuration conf = new Configuration();
+    Configuration conf = getBaseConf();
     final Path jksPath = new Path(testDir.toString(), "test.jks");
     final String ourUrl =
         JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
@@ -283,10 +272,10 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
 
     // ensure that we get nulls when the key isn't there
-    assertEquals(null, provider.getCredentialEntry(
+    assertNull(provider.getCredentialEntry(
         LdapGroupsMapping.BIND_PASSWORD_KEY));
-    assertEquals(null, provider.getCredentialEntry
-        (LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
+    assertNull(provider.getCredentialEntry(
+        LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
 
     // create new aliases
     try {
@@ -350,9 +339,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      final Configuration conf = new Configuration();
-      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
-          "ldap://localhost:" + serverSock.getLocalPort());
+      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+      final Configuration conf = getBaseConf(ldapUrl, null);
       conf.setInt(CONNECTION_TIMEOUT, connectionTimeoutMs);
       mapping.setConf(conf);
 
@@ -406,9 +394,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      final Configuration conf = new Configuration();
-      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
-          "ldap://localhost:" + serverSock.getLocalPort());
+      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+      final Configuration conf = getBaseConf(ldapUrl, null);
       conf.setInt(READ_TIMEOUT, readTimeoutMs);
       mapping.setConf(conf);
 
@@ -436,15 +423,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
    */
   @Test(timeout = 10000)
   public void testSetConf() throws Exception {
-    Configuration conf = new Configuration();
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     Configuration mockConf = Mockito.spy(conf);
     when(mockConf.getPassword(anyString()))
         .thenThrow(new IOException("injected IOException"));
-    // Set a dummy LDAP server URL.
-    mockConf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(mockConf);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
index 51d3673..2469b4b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
@@ -18,10 +18,15 @@
 
 package org.apache.hadoop.security;
 
-import static org.mockito.Mockito.doReturn;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_DEFAULT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import javax.naming.Context;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
@@ -31,13 +36,17 @@ import javax.naming.directory.BasicAttributes;
 import javax.naming.directory.DirContext;
 import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
+import javax.naming.spi.InitialContextFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.junit.Before;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
 
+import java.util.Hashtable;
+
 public class TestLdapGroupsMappingBase {
   @Mock
   private DirContext context;
@@ -62,7 +71,6 @@ public class TestLdapGroupsMappingBase {
   public void setupMocksBase() throws NamingException {
     MockitoAnnotations.initMocks(this);
     DirContext ctx = getContext();
-    doReturn(ctx).when(groupsMapping).getDirContext();
 
     when(ctx.search(Mockito.anyString(), Mockito.anyString(),
         Mockito.any(Object[].class), Mockito.any(SearchControls.class))).
@@ -117,9 +125,30 @@ public class TestLdapGroupsMappingBase {
         thenReturn("CN=some_group,DC=test,DC=com");
   }
 
+  protected Configuration getBaseConf() {
+    return getBaseConf("ldap://test");
+  }
+
+  protected Configuration getBaseConf(String ldapUrl) {
+    return getBaseConf(ldapUrl, getContext());
+  }
+
+  protected Configuration getBaseConf(
+      String ldapUrl, DirContext contextToReturn) {
+    DummyLdapCtxFactory.setContextToReturn(contextToReturn);
+    DummyLdapCtxFactory.setExpectedLdapUrl(ldapUrl);
+
+    Configuration conf = new Configuration();
+    conf.set(LDAP_URL_KEY, ldapUrl);
+    conf.setClass(LDAP_CTX_FACTORY_CLASS_KEY, DummyLdapCtxFactory.class,
+        InitialContextFactory.class);
+    return conf;
+  }
+
   protected DirContext getContext() {
     return context;
   }
+
   protected NamingEnumeration<SearchResult> getUserNames() {
     return userNames;
   }
@@ -139,13 +168,56 @@ public class TestLdapGroupsMappingBase {
   protected LdapGroupsMapping getGroupsMapping() {
     return groupsMapping;
   }
+
   protected String[] getTestGroups() {
     return testGroups;
   }
+
   protected NamingEnumeration getParentGroupNames() {
     return parentGroupNames;
   }
+
   protected String[] getTestParentGroups() {
     return testParentGroups;
   }
+
+  /**
+   * Ldap Context Factory implementation to be used for testing to check
+   * contexts are requested for the expected LDAP server URLs etc.
+   */
+  public static class DummyLdapCtxFactory implements InitialContextFactory {
+
+    private static DirContext contextToReturn;
+    private static String expectedLdapUrl;
+
+    public DummyLdapCtxFactory() {
+    }
+
+    protected static void setContextToReturn(DirContext ctx) {
+      contextToReturn = ctx;
+    }
+
+    protected static void setExpectedLdapUrl(String url) {
+      expectedLdapUrl = url;
+    }
+
+    @Override
+    public Context getInitialContext(Hashtable<?, ?> env)
+        throws NamingException {
+      if (expectedLdapUrl != null) {
+        String actualLdapUrl = (String) env.get(Context.PROVIDER_URL);
+        assertEquals(expectedLdapUrl, actualLdapUrl);
+      }
+      if (contextToReturn == null) {
+        InitialContextFactory defaultFactory = null;
+        try {
+          defaultFactory = LDAP_CTX_FACTORY_CLASS_DEFAULT.newInstance();
+        } catch (ReflectiveOperationException e) {
+          fail("Could not initialize the default factory");
+        }
+        return defaultFactory.getInitialContext(env);
+      }
+      return contextToReturn;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
new file mode 100644
index 0000000..976bfc8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import javax.naming.CommunicationException;
+import javax.naming.directory.SearchControls;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test failover functionality for switching to different
+ * LDAP server URLs upon failures.
+ */
+public class TestLdapGroupsMappingWithFailover
+    extends TestLdapGroupsMappingBase {
+
+  private static final String TEST_USER_NAME = "some_user";
+
+  /**
+   * Test that when disabled, we will retry the configured number
+   * of times using the same LDAP server.
+   */
+  @Test
+  public void testDoesNotFailoverWhenDisabled() throws Exception {
+    final int numAttempts = 3;
+    Configuration conf = getBaseConf();
+    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+        "ldap://test2");
+    DummyLdapCtxFactory.setExpectedLdapUrl("ldap://test");
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY, numAttempts);
+
+    when(getContext().search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class))).thenThrow(new CommunicationException());
+
+    LdapGroupsMapping groupsMapping = getGroupsMapping();
+    groupsMapping.setConf(conf);
+
+    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+    assertTrue(groups.isEmpty());
+
+    // Test that we made 3 attempts using the same server
+    verify(getContext(), times(numAttempts)).search(anyString(), anyString(),
+        any(Object[].class), any(SearchControls.class));
+  }
+
+  /**
+   * Test that when configured, we will make the specified amount of
+   * attempts using one ldap url before failing over to the next one.
+   *
+   * This also tests that we wrap back to the first server
+   * if we've tried them all.
+   */
+  @Test
+  public void testFailover() throws Exception {
+    final Queue<String> ldapUrls = new LinkedList<>();
+    ldapUrls.add("ldap://test");
+    ldapUrls.add("ldap://test1");
+    ldapUrls.add("ldap://test2");
+
+    final int numAttempts = 12;
+    final int numAttemptsBeforeFailover = 2;
+
+    Configuration conf = getBaseConf();
+    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+        "ldap://test2");
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+        numAttemptsBeforeFailover);
+
+    // Set the first expected url and add it back to the queue
+    String nextLdapUrl = ldapUrls.remove();
+    DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+    ldapUrls.add(nextLdapUrl);
+
+    // Number of attempts using a single ldap server url
+    final AtomicInteger serverAttempts = new AtomicInteger(
+        numAttemptsBeforeFailover);
+
+    when(getContext().search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class))).thenAnswer(new Answer<Object>() {
+          @Override
+          public Object answer(InvocationOnMock invocationOnMock)
+              throws Throwable {
+            if (serverAttempts.get() == 1) {
+              // Take the head of the queue and re-queue it to the back
+              String nextLdapUrl = ldapUrls.remove();
+              ldapUrls.add(nextLdapUrl);
+
+              DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+              serverAttempts.set(numAttemptsBeforeFailover);
+            } else {
+              serverAttempts.decrementAndGet();
+            }
+            throw new CommunicationException();
+          }
+        });
+
+    LdapGroupsMapping groupsMapping = getGroupsMapping();
+    groupsMapping.setConf(conf);
+
+    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+    assertTrue(groups.isEmpty());
+
+    // Test that we made 6 attempts overall
+    verify(getContext(), times(numAttempts)).search(anyString(),
+        anyString(), any(Object[].class), any(SearchControls.class));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
index e5cd2b6..f51a944 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.security;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -68,7 +67,7 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // given a user whose ldap query returns a user object with three "memberOf"
     // properties, return an array of strings representing its groups.
     String[] testGroups = new String[] {"abc", "xyz", "sss"};
@@ -76,10 +75,9 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   private void doTestGetGroups(List<String> expectedGroups)
-      throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      throws NamingException {
+    String ldapUrl = "ldap://test";
+    Configuration conf = getBaseConf(ldapUrl);
     // enable single-query lookup
     conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
 
@@ -92,9 +90,7 @@ public class TestLdapGroupsMappingWithOneQuery
     Assert.assertEquals(expectedGroups, groups);
 
     // We should have only made one query because single-query lookup is enabled
-    verify(getContext(), times(1)).search(anyString(),
-        anyString(),
-        any(Object[].class),
-        any(SearchControls.class));
+    verify(getContext(), times(1)).search(anyString(), anyString(),
+        any(Object[].class), any(SearchControls.class));
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/46f0f4c1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
index 17d28a5..4a21345 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
@@ -28,7 +28,6 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -62,7 +61,7 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), contains("posix"),
@@ -73,10 +72,9 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
-      throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      throws NamingException {
+    String ldapUrl = "ldap://test";
+    Configuration conf = getBaseConf(ldapUrl);
     conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY,
         "(objectClass=posixGroup)(cn={0})");
     conf.set(LdapGroupsMapping.USER_SEARCH_FILTER_KEY,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[15/50] [abbrv] hadoop git commit: HDFS-13802. RBF: Remove FSCK from Router Web UI. Contributed by Fei Hui.

Posted by jh...@apache.org.
HDFS-13802. RBF: Remove FSCK from Router Web UI. Contributed by Fei Hui.

(cherry picked from commit f63ee083db7c261207b726f0f44be2e5a92124f3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e5d0528
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e5d0528
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e5d0528

Branch: refs/heads/branch-2-jhung-test
Commit: 6e5d0528a48e18e5848e5240782f823589d15fa1
Parents: ec9a67f
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Oct 12 10:10:12 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html   | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e5d0528/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
index 37fcb92..cf19c94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -44,7 +44,6 @@
         <li><a href="jmx">Metrics</a></li>
         <li><a href="conf">Configuration</a></li>
         <li><a href="logs">Logs</a></li>
-        <li><a href="fsck">FSCK</a></li>
       </ul>
     </li>
   </ul>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[18/50] [abbrv] hadoop git commit: HADOOP-15850. CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0. Contributed by Ted Yu.

Posted by jh...@apache.org.
HADOOP-15850. CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0. Contributed by Ted Yu.

Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
(cherry picked from commit e2cecb681e2aab8b7c5465719cac53dce407a64c)
(cherry picked from commit a7dd244a4995cbf20feb054530fc5913ccf384ac)

	modified:   hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
(cherry picked from commit 84726679f0441bfa0788ad5a5ac2bfa4353afedb)

(cherry picked from commit 8360d15abc29e9ed5f7a2117ab6ac882248e8458)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54757006
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54757006
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54757006

Branch: refs/heads/branch-2-jhung-test
Commit: 547570063e2e6597dbeb45c242915b2b3038c4e7
Parents: 645b311
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Fri Oct 19 13:56:23 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/tools/mapred/CopyCommitter.java   | 8 +++++++-
 .../org/apache/hadoop/tools/mapred/TestCopyCommitter.java    | 4 ----
 2 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/54757006/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
index 6ddaab9..2b32af1 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
@@ -68,6 +68,7 @@ public class CopyCommitter extends FileOutputCommitter {
   private boolean overwrite = false;
   private boolean targetPathExists = true;
   private boolean ignoreFailures = false;
+  private int blocksPerChunk = 0;
 
   /**
    * Create a output committer
@@ -78,6 +79,9 @@ public class CopyCommitter extends FileOutputCommitter {
    */
   public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
     super(outputPath, context);
+    blocksPerChunk = context.getConfiguration().getInt(
+        DistCpOptionSwitch.BLOCKS_PER_CHUNK.getConfigLabel(), 0);
+    LOG.debug("blocks per chunk " + blocksPerChunk);
     this.taskAttemptContext = context;
   }
 
@@ -92,7 +96,9 @@ public class CopyCommitter extends FileOutputCommitter {
     ignoreFailures = conf.getBoolean(
         DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
 
-    concatFileChunks(conf);
+    if (blocksPerChunk > 0) {
+      concatFileChunks(conf);
+    }
 
     super.commitJob(jobContext);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/54757006/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
index 2452d6f..ec5d8ed 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
@@ -81,10 +81,6 @@ public class TestCopyCommitter {
   @Before
   public void createMetaFolder() {
     config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
-    // Unset listing file path since the config is shared by
-    // multiple tests, and some test doesn't set it, such as
-    // testNoCommitAction, but the distcp code will check it.
-    config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
     Path meta = new Path("/meta");
     try {
       cluster.getFileSystem().mkdirs(meta);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[40/50] [abbrv] hadoop git commit: Revert "delete maxpermsize for testing"

Posted by jh...@apache.org.
Revert "delete maxpermsize for testing"

This reverts commit 04be9328f6c01a3c80ea332fd3d1b93af7ce05a0.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2bd1c96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2bd1c96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2bd1c96

Branch: refs/heads/branch-2-jhung-test
Commit: b2bd1c960a4fa80e5a644930395d07581585bff3
Parents: 6669c8b
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Dec 14 15:59:55 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2bd1c96/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 68d9826..288891c 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -112,7 +112,7 @@
     <enforced.maven.version>[3.3.0,)</enforced.maven.version>
 
     <!-- Plugin versions and config -->
-    <maven-surefire-plugin.argLine>-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
+    <maven-surefire-plugin.argLine>-Xmx2048m -XX:MaxPermSize=768m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
     <maven-surefire-plugin.version>2.21.0</maven-surefire-plugin.version>
     <maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
     <maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[39/50] [abbrv] hadoop git commit: Revert "drop thread/dn counts in various tests to avoid OOM errors"

Posted by jh...@apache.org.
Revert "drop thread/dn counts in various tests to avoid OOM errors"

This reverts commit 48191274e03ce66e130706c092e91ed5e6732890.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aa4c7d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aa4c7d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aa4c7d0

Branch: refs/heads/branch-2-jhung-test
Commit: 8aa4c7d0a6060fd1c9af5168f854771b0c2a3e07
Parents: b2bd1c9
Author: Jonathan Hung <jy...@gmail.com>
Authored: Fri Dec 14 16:00:00 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdfs/TestDatanodeDeath.java  |  2 +-
 .../java/org/apache/hadoop/hdfs/TestFileAppend2.java    |  2 +-
 .../hadoop/hdfs/server/balancer/TestBalancer.java       |  2 +-
 .../hadoop/hdfs/server/datanode/TestBatchIbr.java       |  2 +-
 .../TestBlockPlacementPolicyRackFaultTolerant.java      | 12 ++++++------
 5 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
index 36fe966..cd82b2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeDeath.java
@@ -52,7 +52,7 @@ public class TestDatanodeDeath {
   static final int blockSize = 8192;
   static final int numBlocks = 2;
   static final int fileSize = numBlocks * blockSize + 1;
-  static final int numDatanodes = 5;
+  static final int numDatanodes = 15;
   static final short replication = 3;
 
   final int numberOfFiles = 3;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
index ff298a3..cd1b851 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend2.java
@@ -61,7 +61,7 @@ public class TestFileAppend2 {
 
   private byte[] fileContents = null;
 
-  final int numDatanodes = 5;
+  final int numDatanodes = 6;
   final int numberOfFiles = 50;
   final int numThreads = 10;
   final int numAppendsPerThread = 20;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index f1fcfad..2979111 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1966,7 +1966,7 @@ public class TestBalancer {
     initConf(conf);
     conf.setInt(DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, 30);
 
-    int numDNs = 5;
+    int numDNs = 20;
     long[] capacities = new long[numDNs];
     String[] racks = new String[numDNs];
     for(int i = 0; i < numDNs; i++) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
index 94c8882..38c8a38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBatchIbr.java
@@ -59,7 +59,7 @@ public class TestBatchIbr {
   private static final int BLOCK_SIZE = 1024;
   private static final int MAX_BLOCK_NUM = 8;
   private static final int NUM_FILES = 1000;
-  private static final int NUM_THREADS = 64;
+  private static final int NUM_THREADS = 128;
 
   private static final ThreadLocalBuffer IO_BUF = new ThreadLocalBuffer();
   private static final ThreadLocalBuffer VERIFY_BUF = new ThreadLocalBuffer();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aa4c7d0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
index 1834014..f40c464 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
@@ -56,7 +56,7 @@ public class TestBlockPlacementPolicyRackFaultTolerant {
     Configuration conf = new HdfsConfiguration();
     final ArrayList<String> rackList = new ArrayList<String>();
     final ArrayList<String> hostList = new ArrayList<String>();
-    for (int i = 0; i < 3; i++) {
+    for (int i = 0; i < 10; i++) {
       for (int j = 0; j < 2; j++) {
         rackList.add("/rack" + i);
         hostList.add("/host" + i + j);
@@ -96,8 +96,8 @@ public class TestBlockPlacementPolicyRackFaultTolerant {
   private void doTestChooseTargetNormalCase() throws Exception {
     String clientMachine = "client.foo.com";
     short[][] testSuite = {
-        {3, 2}//, {3, 7}, {3, 8}, {3, 10}, {9, 1}, {10, 1}, {10, 6}, {11, 6},
-        //{11, 9}
+        {3, 2}, {3, 7}, {3, 8}, {3, 10}, {9, 1}, {10, 1}, {10, 6}, {11, 6},
+        {11, 9}
     };
     // Test 5 files
     int fileCount = 0;
@@ -139,12 +139,12 @@ public class TestBlockPlacementPolicyRackFaultTolerant {
     // Create the file with client machine
     HdfsFileStatus fileStatus = namesystem.startFile(src, perm,
         clientMachine, clientMachine, EnumSet.of(CreateFlag.CREATE), true,
-        (short) 5, DEFAULT_BLOCK_SIZE, null, false);
+        (short) 20, DEFAULT_BLOCK_SIZE, null, false);
 
     //test chooseTarget for new file
     LocatedBlock locatedBlock = nameNodeRpc.addBlock(src, clientMachine,
         null, null, fileStatus.getFileId(), null, null);
-    doTestLocatedBlock(5, locatedBlock);
+    doTestLocatedBlock(20, locatedBlock);
 
     DatanodeInfo[] locs = locatedBlock.getLocations();
     String[] storageIDs = locatedBlock.getStorageIDs();
@@ -156,7 +156,7 @@ public class TestBlockPlacementPolicyRackFaultTolerant {
         String[] partStorageIDs = new String[i];
         System.arraycopy(locs, 0, partLocs, 0, i);
         System.arraycopy(storageIDs, 0, partStorageIDs, 0, i);
-        for (int j = 1; j < 5 - i; j++) {
+        for (int j = 1; j < 20 - i; j++) {
           LocatedBlock additionalLocatedBlock =
               nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(),
                   locatedBlock.getBlock(), partLocs,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[22/50] [abbrv] hadoop git commit: HADOOP-15868. AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy. Contributed by Jinhu Wu.

Posted by jh...@apache.org.
HADOOP-15868. AliyunOSS: update document for properties of multiple part download, multiple part upload and directory copy. Contributed by Jinhu Wu.

(cherry picked from commit 7574d18538e838f40581519080d7c8621c65e53b)
(cherry picked from commit 366541d834f70fd6f8d4c5296a9e844236c6fd74)
(cherry picked from commit c5a227062fae33ce3137161802d6f090a6af010e)
(cherry picked from commit ca22bf175f91980f78dedf1cd27f114acd56fa50)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/091c7cfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/091c7cfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/091c7cfc

Branch: refs/heads/branch-2-jhung-test
Commit: 091c7cfc326445fbe2f52563f769b88a438c9fe8
Parents: 5475700
Author: Sammi Chen <sa...@intel.com>
Authored: Fri Oct 26 15:19:56 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../site/markdown/tools/hadoop-aliyun/index.md  | 36 ++++++++++++++++++++
 1 file changed, 36 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/091c7cfc/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 0703790..0c3131d 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -229,6 +229,42 @@ please raise your issues with them.
     </property>
 
     <property>
+      <name>fs.oss.upload.active.blocks</name>
+      <value>4</value>
+      <description>Active(Concurrent) upload blocks when uploading a file.</description>
+    </property>
+
+    <property>
+      <name>fs.oss.multipart.download.threads</name>
+      <value>10</value>
+      <description>The maximum number of threads allowed in the pool for multipart download and upload.</description>
+    </property>
+
+    <property>
+      <name>fs.oss.multipart.download.ahead.part.max.number</name>
+      <value>4</value>
+      <description>The maximum number of read ahead parts when reading a file.</description>
+    </property>
+
+    <property>
+      <name>fs.oss.max.total.tasks</name>
+      <value>128</value>
+      <description>The maximum queue number for multipart download and upload.</description>
+    </property>
+
+    <property>
+      <name>fs.oss.max.copy.threads</name>
+      <value>25</value>
+      <description>The maximum number of threads allowed in the pool for copy operations.</description>
+    </property>
+
+    <property>
+      <name>fs.oss.max.copy.tasks.per.dir</name>
+      <value>5</value>
+      <description>The maximum number of concurrent tasks allowed when copying a directory.</description>
+    </property>
+
+    <property>
       <name>fs.oss.multipart.upload.threshold</name>
       <value>20971520</value>
       <description>Minimum size in bytes before we start a multipart uploads or copy.</description>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[48/50] [abbrv] hadoop git commit: HADOOP-15930. Exclude MD5 checksum files from release artifact.

Posted by jh...@apache.org.
HADOOP-15930. Exclude MD5 checksum files from release artifact.

(cherry picked from commit df5e863fee544c9283e28a21c2788c008d7e3e04)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/914716d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/914716d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/914716d1

Branch: refs/heads/branch-2-jhung-test
Commit: 914716d15c5dfe901dc1674e8dc771e00e8091e4
Parents: 0f9757f
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Nov 14 14:50:34 2018 +0900
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 dev-support/bin/create-release | 14 --------------
 1 file changed, 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/914716d1/dev-support/bin/create-release
----------------------------------------------------------------------
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index fbe3fb2..33c87cd 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -167,11 +167,6 @@ function run()
   fi
 }
 
-function domd5()
-{
-  run "${MD5SUM}" "${1}" > "${1}.md5"
-}
-
 ## @description  set JAVA_HOME properly
 ## @audience     public
 ## @stability    unstable
@@ -241,11 +236,6 @@ function set_defaults
     fi
   fi
 
-  MD5SUM=$(command -v md5sum)
-  if [[ -z "${MD5SUM}" ]]; then
-    MD5SUM=$(command -v md5)
-  fi
-
   NATIVE=false
   OSNAME=$(uname -s)
 
@@ -601,9 +591,6 @@ function signartifacts
   declare i
 
   if [[ "${SIGN}" = false ]]; then
-    for i in ${ARTIFACTS_DIR}/*; do
-      domd5 "${i}"
-    done
     echo ""
     echo "Remember to sign the artifacts before staging them on the open"
     echo ""
@@ -615,7 +602,6 @@ function signartifacts
   for i in ${ARTIFACTS_DIR}/*; do
     ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
     ${GPG} --print-mds "${i}" > "${i}.mds"
-    domd5 "${i}"
   done
 
   if [[ "${ASFRELEASE}" = true ]]; then


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[45/50] [abbrv] hadoop git commit: HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak.

Posted by jh...@apache.org.
HADOOP-15950. Failover for LdapGroupsMapping. Contributed by Lukas Majercak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f8c879d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f8c879d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f8c879d

Branch: refs/heads/branch-2-jhung-test
Commit: 1f8c879d169cce8855f6865939fb9f14b147c6b4
Parents: 15ce8df
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Mon Dec 3 12:10:05 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop/security/LdapGroupsMapping.java      | 163 +++++++++++++------
 .../src/main/resources/core-default.xml         |  28 +++-
 .../src/site/markdown/GroupsMapping.md          |  54 +++++-
 .../hadoop/security/TestLdapGroupsMapping.java  |  80 ++++-----
 .../security/TestLdapGroupsMappingBase.java     |  76 ++++++++-
 .../TestLdapGroupsMappingWithFailover.java      | 142 ++++++++++++++++
 .../TestLdapGroupsMappingWithOneQuery.java      |  16 +-
 .../TestLdapGroupsMappingWithPosixGroup.java    |  10 +-
 8 files changed, 450 insertions(+), 119 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index 6beaa9e..83eb5ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -25,6 +25,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Hashtable;
+import java.util.Iterator;
 import java.util.List;
 import java.util.HashSet;
 import java.util.Collection;
@@ -40,7 +41,10 @@ import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
 import javax.naming.ldap.LdapName;
 import javax.naming.ldap.Rdn;
+import javax.naming.spi.InitialContextFactory;
 
+import com.google.common.collect.Iterators;
+import com.sun.jndi.ldap.LdapCtxFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configurable;
@@ -83,7 +87,7 @@ public class LdapGroupsMapping
   public static final String LDAP_CONFIG_PREFIX = "hadoop.security.group.mapping.ldap";
 
   /*
-   * URL of the LDAP server
+   * URL of the LDAP server(s)
    */
   public static final String LDAP_URL_KEY = LDAP_CONFIG_PREFIX + ".url";
   public static final String LDAP_URL_DEFAULT = "";
@@ -232,6 +236,20 @@ public class LdapGroupsMapping
       LDAP_CONFIG_PREFIX + ".read.timeout.ms";
   public static final int READ_TIMEOUT_DEFAULT = 60 * 1000; // 60 seconds
 
+  public static final String LDAP_NUM_ATTEMPTS_KEY =
+      LDAP_CONFIG_PREFIX + ".num.attempts";
+  public static final int LDAP_NUM_ATTEMPTS_DEFAULT = 3;
+
+  public static final String LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY =
+      LDAP_CONFIG_PREFIX + ".num.attempts.before.failover";
+  public static final int LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT =
+      LDAP_NUM_ATTEMPTS_DEFAULT;
+
+  public static final String LDAP_CTX_FACTORY_CLASS_KEY =
+      LDAP_CONFIG_PREFIX + ".ctx.factory.class";
+  public static final Class<? extends LdapCtxFactory>
+      LDAP_CTX_FACTORY_CLASS_DEFAULT = LdapCtxFactory.class;
+
   private static final Logger LOG =
       LoggerFactory.getLogger(LdapGroupsMapping.class);
 
@@ -242,8 +260,10 @@ public class LdapGroupsMapping
 
   private DirContext ctx;
   private Configuration conf;
-  
-  private String ldapUrl;
+
+  private Iterator<String> ldapUrls;
+  private String currentLdapUrl;
+
   private boolean useSsl;
   private String keystore;
   private String keystorePass;
@@ -258,14 +278,15 @@ public class LdapGroupsMapping
   private String memberOfAttr;
   private String groupMemberAttr;
   private String groupNameAttr;
-  private int    groupHierarchyLevels;
+  private int groupHierarchyLevels;
   private String posixUidAttr;
   private String posixGidAttr;
   private boolean isPosix;
   private boolean useOneQuery;
+  private int numAttempts;
+  private int numAttemptsBeforeFailover;
+  private Class<? extends InitialContextFactory> ldapCxtFactoryClass;
 
-  public static final int RECONNECT_RETRY_COUNT = 3;
-  
   /**
    * Returns list of groups for a user.
    * 
@@ -279,20 +300,31 @@ public class LdapGroupsMapping
   @Override
   public synchronized List<String> getGroups(String user) {
     /*
-     * Normal garbage collection takes care of removing Context instances when they are no longer in use. 
-     * Connections used by Context instances being garbage collected will be closed automatically.
-     * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. 
+     * Normal garbage collection takes care of removing Context instances when
+     * they are no longer in use. Connections used by Context instances being
+     * garbage collected will be closed automatically. So in case connection is
+     * closed and gets CommunicationException, retry some times with new new
+     * DirContext/connection.
      */
-    for(int retry = 0; retry < RECONNECT_RETRY_COUNT; retry++) {
+
+    // Tracks the number of attempts made using the same LDAP server
+    int atemptsBeforeFailover = 1;
+
+    for (int attempt = 1; attempt <= numAttempts; attempt++,
+        atemptsBeforeFailover++) {
       try {
         return doGetGroups(user, groupHierarchyLevels);
       } catch (NamingException e) {
-        LOG.warn("Failed to get groups for user " + user + " (retry=" + retry
-            + ") by " + e);
+        LOG.warn("Failed to get groups for user {} (attempt={}/{}) using {}. " +
+            "Exception: ", user, attempt, numAttempts, currentLdapUrl, e);
         LOG.trace("TRACE", e);
+
+        if (failover(atemptsBeforeFailover, numAttemptsBeforeFailover)) {
+          atemptsBeforeFailover = 0;
+        }
       }
 
-      //reset ctx so that new DirContext can be created with new connection
+      // Reset ctx so that new DirContext can be created with new connection
       this.ctx = null;
     }
     
@@ -378,10 +410,10 @@ public class LdapGroupsMapping
   private List<String> lookupGroup(SearchResult result, DirContext c,
       int goUpHierarchy)
       throws NamingException {
-    List<String> groups = new ArrayList<String>();
-    Set<String> groupDNs = new HashSet<String>();
+    List<String> groups = new ArrayList<>();
+    Set<String> groupDNs = new HashSet<>();
 
-    NamingEnumeration<SearchResult> groupResults = null;
+    NamingEnumeration<SearchResult> groupResults;
     // perform the second LDAP query
     if (isPosix) {
       groupResults = lookupPosixGroup(result, c);
@@ -402,10 +434,10 @@ public class LdapGroupsMapping
       }
       if (goUpHierarchy > 0 && !isPosix) {
         // convert groups to a set to ensure uniqueness
-        Set<String> groupset = new HashSet<String>(groups);
+        Set<String> groupset = new HashSet<>(groups);
         goUpGroupHierarchy(groupDNs, goUpHierarchy, groupset);
         // convert set back to list for compatibility
-        groups = new ArrayList<String>(groupset);
+        groups = new ArrayList<>(groupset);
       }
     }
     return groups;
@@ -433,11 +465,9 @@ public class LdapGroupsMapping
         userSearchFilter, new Object[]{user}, SEARCH_CONTROLS);
     // return empty list if the user can not be found.
     if (!results.hasMoreElements()) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("doGetGroups(" + user + ") returned no groups because the " +
-            "user is not found.");
-      }
-      return new ArrayList<String>();
+      LOG.debug("doGetGroups({}) returned no groups because the " +
+          "user is not found.", user);
+      return new ArrayList<>();
     }
     SearchResult result = results.nextElement();
 
@@ -455,7 +485,7 @@ public class LdapGroupsMapping
               memberOfAttr + "' attribute." +
               "Returned user object: " + result.toString());
         }
-        groups = new ArrayList<String>();
+        groups = new ArrayList<>();
         NamingEnumeration groupEnumeration = groupDNAttr.getAll();
         while (groupEnumeration.hasMore()) {
           String groupDN = groupEnumeration.next().toString();
@@ -470,9 +500,7 @@ public class LdapGroupsMapping
     if (groups == null || groups.isEmpty() || goUpHierarchy > 0) {
       groups = lookupGroup(result, c, goUpHierarchy);
     }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("doGetGroups(" + user + ") returned " + groups);
-    }
+    LOG.debug("doGetGroups({}) returned {}", user, groups);
     return groups;
   }
 
@@ -480,7 +508,7 @@ public class LdapGroupsMapping
   */
   void getGroupNames(SearchResult groupResult, Collection<String> groups,
                      Collection<String> groupDNs, boolean doGetDNs)
-                     throws NamingException  {
+                     throws NamingException {
     Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
     if (groupName == null) {
       throw new NamingException("The group object does not have " +
@@ -517,7 +545,7 @@ public class LdapGroupsMapping
       return;
     }
     DirContext context = getDirContext();
-    Set<String> nextLevelGroups = new HashSet<String>();
+    Set<String> nextLevelGroups = new HashSet<>();
     StringBuilder filter = new StringBuilder();
     filter.append("(&").append(groupSearchFilter).append("(|");
     for (String dn : groupDNs) {
@@ -537,13 +565,32 @@ public class LdapGroupsMapping
     goUpGroupHierarchy(nextLevelGroups, goUpHierarchy - 1, groups);
   }
 
-  DirContext getDirContext() throws NamingException {
+  /**
+   * Check whether we should fail over to the next LDAP server.
+   * @param attemptsMadeWithSameLdap current number of attempts made
+   *                                 with using same LDAP instance
+   * @param maxAttemptsBeforeFailover maximum number of attempts
+   *                                  before failing over
+   * @return true if we should fail over to the next LDAP server
+   */
+  protected boolean failover(
+      int attemptsMadeWithSameLdap, int maxAttemptsBeforeFailover) {
+    if (attemptsMadeWithSameLdap >= maxAttemptsBeforeFailover) {
+      String previousLdapUrl = currentLdapUrl;
+      currentLdapUrl = ldapUrls.next();
+      LOG.info("Reached {} attempts on {}, failing over to {}",
+          attemptsMadeWithSameLdap, previousLdapUrl, currentLdapUrl);
+      return true;
+    }
+    return false;
+  }
+
+  private DirContext getDirContext() throws NamingException {
     if (ctx == null) {
       // Set up the initial environment for LDAP connectivity
-      Hashtable<String, String> env = new Hashtable<String, String>();
-      env.put(Context.INITIAL_CONTEXT_FACTORY,
-          com.sun.jndi.ldap.LdapCtxFactory.class.getName());
-      env.put(Context.PROVIDER_URL, ldapUrl);
+      Hashtable<String, String> env = new Hashtable<>();
+      env.put(Context.INITIAL_CONTEXT_FACTORY, ldapCxtFactoryClass.getName());
+      env.put(Context.PROVIDER_URL, currentLdapUrl);
       env.put(Context.SECURITY_AUTHENTICATION, "simple");
 
       // Set up SSL security, if necessary
@@ -581,7 +628,7 @@ public class LdapGroupsMapping
    * Caches groups, no need to do that for this provider
    */
   @Override
-  public void cacheGroupsRefresh() throws IOException {
+  public void cacheGroupsRefresh() {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -591,7 +638,7 @@ public class LdapGroupsMapping
    * @param groups unused
    */
   @Override
-  public void cacheGroupsAdd(List<String> groups) throws IOException {
+  public void cacheGroupsAdd(List<String> groups) {
     // does nothing in this provider of user to groups mapping
   }
 
@@ -602,10 +649,12 @@ public class LdapGroupsMapping
 
   @Override
   public synchronized void setConf(Configuration conf) {
-    ldapUrl = conf.get(LDAP_URL_KEY, LDAP_URL_DEFAULT);
-    if (ldapUrl == null || ldapUrl.isEmpty()) {
-      throw new RuntimeException("LDAP URL is not configured");
+    String[] urls = conf.getStrings(LDAP_URL_KEY, LDAP_URL_DEFAULT);
+    if (urls == null || urls.length == 0) {
+      throw new RuntimeException("LDAP URL(s) are not configured");
     }
+    ldapUrls = Iterators.cycle(urls);
+    currentLdapUrl = ldapUrls.next();
 
     useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
     if (useSsl) {
@@ -621,17 +670,13 @@ public class LdapGroupsMapping
     
     String baseDN = conf.getTrimmed(BASE_DN_KEY, BASE_DN_DEFAULT);
 
-    //User search base which defaults to base dn.
+    // User search base which defaults to base dn.
     userbaseDN = conf.getTrimmed(USER_BASE_DN_KEY, baseDN);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Usersearch baseDN: " + userbaseDN);
-    }
+    LOG.debug("Usersearch baseDN: {}", userbaseDN);
 
-    //Group search base which defaults to base dn.
+    // Group search base which defaults to base dn.
     groupbaseDN = conf.getTrimmed(GROUP_BASE_DN_KEY, baseDN);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Groupsearch baseDN: " + userbaseDN);
-    }
+    LOG.debug("Groupsearch baseDN: {}", groupbaseDN);
 
     groupSearchFilter =
         conf.get(GROUP_SEARCH_FILTER_KEY, GROUP_SEARCH_FILTER_DEFAULT);
@@ -655,7 +700,8 @@ public class LdapGroupsMapping
     posixGidAttr =
         conf.get(POSIX_GID_ATTR_KEY, POSIX_GID_ATTR_DEFAULT);
 
-    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT, DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
+    int dirSearchTimeout = conf.getInt(DIRECTORY_SEARCH_TIMEOUT,
+        DIRECTORY_SEARCH_TIMEOUT_DEFAULT);
     SEARCH_CONTROLS.setTimeLimit(dirSearchTimeout);
     // Limit the attributes returned to only those required to speed up the search.
     // See HADOOP-10626 and HADOOP-12001 for more details.
@@ -669,9 +715,26 @@ public class LdapGroupsMapping
     }
     SEARCH_CONTROLS.setReturningAttributes(returningAttributes);
 
+    ldapCxtFactoryClass = conf.getClass(LDAP_CTX_FACTORY_CLASS_KEY,
+        LDAP_CTX_FACTORY_CLASS_DEFAULT, InitialContextFactory.class);
+
+    this.numAttempts = conf.getInt(LDAP_NUM_ATTEMPTS_KEY,
+        LDAP_NUM_ATTEMPTS_DEFAULT);
+    this.numAttemptsBeforeFailover = conf.getInt(
+        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+        LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_DEFAULT);
+
     this.conf = conf;
   }
 
+  /**
+   * Get URLs of configured LDAP servers.
+   * @return URLs of LDAP servers being used.
+   */
+  public Iterator<String> getLdapUrls() {
+    return ldapUrls;
+  }
+
   private void loadSslConf(Configuration sslConf) {
     keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
     keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY,
@@ -721,8 +784,8 @@ public class LdapGroupsMapping
         password = new String(passchars);
       }
     } catch (IOException ioe) {
-      LOG.warn("Exception while trying to get password for alias " + alias
-              + ": ", ioe);
+      LOG.warn("Exception while trying to get password for alias {}:",
+          alias, ioe);
     }
     return password;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e262b33..e5ce728 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -234,11 +234,35 @@
 </property>
 
 <property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+  <value>3</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations.
+    If this limit is exceeded, LdapGroupsMapping will return an empty
+    group list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+  <value>3</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations
+    using a single LDAP instance. If multiple LDAP servers are configured
+    and this number of failed operations is reached, we will switch to the
+    next LDAP server. The configuration for the overall number of attempts
+    will still be respected, failover will thus be performed only if this
+    property is less than hadoop.security.group.mapping.ldap.num.attempts.
+  </description>
+</property>
+
+<property>
   <name>hadoop.security.group.mapping.ldap.url</name>
   <value></value>
   <description>
-    The URL of the LDAP server to use for resolving user groups when using
-    the LdapGroupsMapping user to group mapping.
+    The URL of the LDAP server(s) to use for resolving user groups when using
+    the LdapGroupsMapping user to group mapping. Supports configuring multiple
+    LDAP servers via a comma-separated list.
   </description>
 </property>
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index c6af930..b957821 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -73,7 +73,7 @@ The cache invalidation is configurable via `hadoop.security.groups.negative-cach
 LDAP Groups Mapping
 --------
 This provider supports LDAP with simple password authentication using JNDI API.
-`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server for resolving user groups.
+`hadoop.security.group.mapping.ldap.url` must be set. This refers to the URL of the LDAP server(s) for resolving user groups. It supports configuring multiple LDAP servers via a comma-separated list.
 
 `hadoop.security.group.mapping.ldap.base` configures the search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.
 Get groups for a given username first looks up the user and then looks up the groups for the user result. If the directory setup has different user and group search bases, use `hadoop.security.group.mapping.ldap.userbase` and `hadoop.security.group.mapping.ldap.groupbase` configs.
@@ -115,6 +115,58 @@ To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memb
 If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`.
 Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`.
 
+### Configuring retries and multiple LDAP servers with failover ###
+If there are issues encountered when retrieving information from LDAP servers, the request will be retried. To configure the number of retries, use the following configuration:
+
+```<property>
+     <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+     <value>3</value>
+     <description>
+       This property is the number of attempts to be made for LDAP operations.
+       If this limit is exceeded, LdapGroupsMapping will return an empty
+       group list.
+     </description>
+    </property>
+```
+
+LDAP Groups Mapping also supports configuring multiple LDAP servers and failover if a particular instance is not available or is misbehaving.
+The following configuration shows configuring 3 LDAP servers. Additionally, 2 attempts will be made for each server before failing over to the next one, with 6 attempts overall before failing.
+
+```
+<property>
+  <name>hadoop.security.group.mapping.ldap.url</name>
+  <value>ldap://server1,ldap://server2,ldap://server3</value>
+  <description>
+    The URL of the LDAP server(s) to use for resolving user groups when using
+    the LdapGroupsMapping user to group mapping. Supports configuring multiple
+    LDAP servers via a comma-separated list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts</name>
+  <value>6</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations.
+    If this limit is exceeded, LdapGroupsMapping will return an empty
+    group list.
+  </description>
+</property>
+
+<property>
+  <name>hadoop.security.group.mapping.ldap.num.attempts.before.failover</name>
+  <value>2</value>
+  <description>
+    This property is the number of attempts to be made for LDAP operations
+    using a single LDAP instance. If multiple LDAP servers are configured
+    and this number of failed operations is reached, we will switch to the
+    next LDAP server. The configuration for the overall number of attempts
+    will still be respected, failover will thus be performed only if this
+    property is less than hadoop.security.group.mapping.ldap.num.attempts.
+  </description>
+</property>
+```
+
 Composite Groups Mapping
 --------
 `CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
index d9f0c2a..a361d04 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java
@@ -18,11 +18,12 @@
 package org.apache.hadoop.security;
 
 import static org.apache.hadoop.security.LdapGroupsMapping.CONNECTION_TIMEOUT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
 import static org.apache.hadoop.security.LdapGroupsMapping.READ_TIMEOUT;
 import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
 import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
@@ -82,14 +83,16 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   private final String userDN = "CN=some_user,DC=test,DC=com";
 
+  private static final String TEST_LDAP_URL = "ldap://test";
+
   @Before
-  public void setupMocks() throws NamingException {
+  public void setupMocks() {
     when(getUserSearchResult().getNameInNamespace()).
         thenReturn(userDN);
   }
   
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -100,9 +103,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDifferentBaseDNs() throws Exception {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     String userBaseDN = "ou=Users,dc=xxx,dc=com ";
     String groupBaseDN = " ou=Groups,dc=xxx,dc=com";
     conf.set(LdapGroupsMapping.USER_BASE_DN_KEY, userBaseDN);
@@ -113,9 +114,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
 
   @Test
   public void testGetGroupsWithDefaultBaseDN() throws Exception {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     String baseDN = " dc=xxx,dc=com ";
     conf.set(LdapGroupsMapping.BASE_DN_KEY, baseDN);
     doTestGetGroupsWithBaseDN(conf, baseDN.trim(), baseDN.trim());
@@ -159,7 +158,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithHierarchy() throws IOException, NamingException {
+  public void testGetGroupsWithHierarchy() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     // The parent search is run once for each level, and is a different search
@@ -175,7 +174,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
+  public void testGetGroupsWithConnectionClosed() throws NamingException {
     // The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
     // then after reconnected return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), anyString(), any(Object[].class),
@@ -190,21 +189,20 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   }
 
   @Test
-  public void testGetGroupsWithLdapDown() throws IOException, NamingException {
+  public void testGetGroupsWithLdapDown() throws NamingException {
     // This mocks the case where Ldap server is down, and always throws CommunicationException 
     when(getContext().search(anyString(), anyString(), any(Object[].class),
         any(SearchControls.class)))
         .thenThrow(new CommunicationException("Connection is closed"));
     
     // Ldap server is down, no groups should be retrieved
-    doTestGetGroups(Arrays.asList(new String[] {}), 
-        LdapGroupsMapping.RECONNECT_RETRY_COUNT);
+    doTestGetGroups(Arrays.asList(new String[] {}), 4);
   }
   
-  private void doTestGetGroups(List<String> expectedGroups, int searchTimes) throws IOException, NamingException {  
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+  private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
+      throws NamingException {
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, searchTimes);
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(conf);
@@ -216,17 +214,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     
     // We should have searched for a user, and then two groups
     verify(getContext(), times(searchTimes)).search(anyString(),
-                                         anyString(),
-                                         any(Object[].class),
-                                         any(SearchControls.class));
+        anyString(), any(Object[].class), any(SearchControls.class));
   }
 
   private void doTestGetGroupsWithParent(List<String> expectedGroups,
-      int searchTimesGroup, int searchTimesParentGroup)
-          throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      int searchTimesGroup, int searchTimesParentGroup) throws NamingException {
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     // Set the config to get parents 1 level up
     conf.setInt(LdapGroupsMapping.GROUP_HIERARCHY_LEVELS_KEY, 1);
 
@@ -237,18 +230,14 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     List<String> groups = groupsMapping.getGroups("some_user");
 
     // compare lists, ignoring the order
-    Assert.assertEquals(new HashSet<String>(expectedGroups),
-        new HashSet<String>(groups));
+    Assert.assertEquals(new HashSet<>(expectedGroups), new HashSet<>(groups));
 
     // We should have searched for a user, and group
     verify(getContext(), times(searchTimesGroup)).search(anyString(),
-                                         anyString(),
-                                         any(Object[].class),
-                                         any(SearchControls.class));
+        anyString(), any(Object[].class), any(SearchControls.class));
     // One groups search for the parent group should have been done
-    verify(getContext(), times(searchTimesParentGroup)).search(anyString(),
-                                         anyString(),
-                                         any(SearchControls.class));
+    verify(getContext(), times(searchTimesParentGroup)).search(
+        anyString(), anyString(), any(SearchControls.class));
   }
 
   @Test
@@ -268,7 +257,7 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
   @Test
   public void testConfGetPassword() throws Exception {
     File testDir = GenericTestUtils.getTestDir();
-    Configuration conf = new Configuration();
+    Configuration conf = getBaseConf();
     final Path jksPath = new Path(testDir.toString(), "test.jks");
     final String ourUrl =
         JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
@@ -283,10 +272,10 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
     char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
 
     // ensure that we get nulls when the key isn't there
-    assertEquals(null, provider.getCredentialEntry(
+    assertNull(provider.getCredentialEntry(
         LdapGroupsMapping.BIND_PASSWORD_KEY));
-    assertEquals(null, provider.getCredentialEntry
-        (LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
+    assertNull(provider.getCredentialEntry(
+        LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY));
 
     // create new aliases
     try {
@@ -350,9 +339,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      final Configuration conf = new Configuration();
-      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
-          "ldap://localhost:" + serverSock.getLocalPort());
+      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+      final Configuration conf = getBaseConf(ldapUrl, null);
       conf.setInt(CONNECTION_TIMEOUT, connectionTimeoutMs);
       mapping.setConf(conf);
 
@@ -406,9 +394,8 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
       ldapServer.start();
 
       final LdapGroupsMapping mapping = new LdapGroupsMapping();
-      final Configuration conf = new Configuration();
-      conf.set(LdapGroupsMapping.LDAP_URL_KEY,
-          "ldap://localhost:" + serverSock.getLocalPort());
+      String ldapUrl = "ldap://localhost:" + serverSock.getLocalPort();
+      final Configuration conf = getBaseConf(ldapUrl, null);
       conf.setInt(READ_TIMEOUT, readTimeoutMs);
       mapping.setConf(conf);
 
@@ -436,15 +423,12 @@ public class TestLdapGroupsMapping extends TestLdapGroupsMappingBase {
    */
   @Test(timeout = 10000)
   public void testSetConf() throws Exception {
-    Configuration conf = new Configuration();
+    Configuration conf = getBaseConf(TEST_LDAP_URL);
     Configuration mockConf = Mockito.spy(conf);
     when(mockConf.getPassword(anyString()))
         .thenThrow(new IOException("injected IOException"));
-    // Set a dummy LDAP server URL.
-    mockConf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
 
     LdapGroupsMapping groupsMapping = getGroupsMapping();
     groupsMapping.setConf(mockConf);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
index 51d3673..2469b4b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingBase.java
@@ -18,10 +18,15 @@
 
 package org.apache.hadoop.security;
 
-import static org.mockito.Mockito.doReturn;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_DEFAULT;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_CTX_FACTORY_CLASS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import javax.naming.Context;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
@@ -31,13 +36,17 @@ import javax.naming.directory.BasicAttributes;
 import javax.naming.directory.DirContext;
 import javax.naming.directory.SearchControls;
 import javax.naming.directory.SearchResult;
+import javax.naming.spi.InitialContextFactory;
 
+import org.apache.hadoop.conf.Configuration;
 import org.junit.Before;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
 
+import java.util.Hashtable;
+
 public class TestLdapGroupsMappingBase {
   @Mock
   private DirContext context;
@@ -62,7 +71,6 @@ public class TestLdapGroupsMappingBase {
   public void setupMocksBase() throws NamingException {
     MockitoAnnotations.initMocks(this);
     DirContext ctx = getContext();
-    doReturn(ctx).when(groupsMapping).getDirContext();
 
     when(ctx.search(Mockito.anyString(), Mockito.anyString(),
         Mockito.any(Object[].class), Mockito.any(SearchControls.class))).
@@ -117,9 +125,30 @@ public class TestLdapGroupsMappingBase {
         thenReturn("CN=some_group,DC=test,DC=com");
   }
 
+  protected Configuration getBaseConf() {
+    return getBaseConf("ldap://test");
+  }
+
+  protected Configuration getBaseConf(String ldapUrl) {
+    return getBaseConf(ldapUrl, getContext());
+  }
+
+  protected Configuration getBaseConf(
+      String ldapUrl, DirContext contextToReturn) {
+    DummyLdapCtxFactory.setContextToReturn(contextToReturn);
+    DummyLdapCtxFactory.setExpectedLdapUrl(ldapUrl);
+
+    Configuration conf = new Configuration();
+    conf.set(LDAP_URL_KEY, ldapUrl);
+    conf.setClass(LDAP_CTX_FACTORY_CLASS_KEY, DummyLdapCtxFactory.class,
+        InitialContextFactory.class);
+    return conf;
+  }
+
   protected DirContext getContext() {
     return context;
   }
+
   protected NamingEnumeration<SearchResult> getUserNames() {
     return userNames;
   }
@@ -139,13 +168,56 @@ public class TestLdapGroupsMappingBase {
   protected LdapGroupsMapping getGroupsMapping() {
     return groupsMapping;
   }
+
   protected String[] getTestGroups() {
     return testGroups;
   }
+
   protected NamingEnumeration getParentGroupNames() {
     return parentGroupNames;
   }
+
   protected String[] getTestParentGroups() {
     return testParentGroups;
   }
+
+  /**
+   * Ldap Context Factory implementation to be used for testing to check
+   * contexts are requested for the expected LDAP server URLs etc.
+   */
+  public static class DummyLdapCtxFactory implements InitialContextFactory {
+
+    private static DirContext contextToReturn;
+    private static String expectedLdapUrl;
+
+    public DummyLdapCtxFactory() {
+    }
+
+    protected static void setContextToReturn(DirContext ctx) {
+      contextToReturn = ctx;
+    }
+
+    protected static void setExpectedLdapUrl(String url) {
+      expectedLdapUrl = url;
+    }
+
+    @Override
+    public Context getInitialContext(Hashtable<?, ?> env)
+        throws NamingException {
+      if (expectedLdapUrl != null) {
+        String actualLdapUrl = (String) env.get(Context.PROVIDER_URL);
+        assertEquals(expectedLdapUrl, actualLdapUrl);
+      }
+      if (contextToReturn == null) {
+        InitialContextFactory defaultFactory = null;
+        try {
+          defaultFactory = LDAP_CTX_FACTORY_CLASS_DEFAULT.newInstance();
+        } catch (ReflectiveOperationException e) {
+          fail("Could not initialize the default factory");
+        }
+        return defaultFactory.getInitialContext(env);
+      }
+      return contextToReturn;
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
new file mode 100644
index 0000000..4f6ea1f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithFailover.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import javax.naming.CommunicationException;
+import javax.naming.directory.SearchControls;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_NUM_ATTEMPTS_KEY;
+import static org.apache.hadoop.security.LdapGroupsMapping.LDAP_URL_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test failover functionality for switching to different
+ * LDAP server URLs upon failures.
+ */
+public class TestLdapGroupsMappingWithFailover
+    extends TestLdapGroupsMappingBase {
+
+  private static final String TEST_USER_NAME = "some_user";
+
+  /**
+   * Test that when disabled, we will retry the configured number
+   * of times using the same LDAP server.
+   */
+  @Test
+  public void testDoesNotFailoverWhenDisabled() throws Exception {
+    final int numAttempts = 3;
+    Configuration conf = getBaseConf();
+    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+        "ldap://test2");
+    DummyLdapCtxFactory.setExpectedLdapUrl("ldap://test");
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY, numAttempts);
+
+    when(getContext().search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class))).thenThrow(new CommunicationException());
+
+    LdapGroupsMapping groupsMapping = getGroupsMapping();
+    groupsMapping.setConf(conf);
+
+    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+    assertTrue(groups.isEmpty());
+
+    // Test that we made 3 attempts using the same server
+    verify(getContext(), times(numAttempts)).search(anyString(), anyString(),
+        any(Object[].class), any(SearchControls.class));
+  }
+
+  /**
+   * Test that when configured, we will make the specified amount of
+   * attempts using one ldap url before failing over to the next one.
+   *
+   * This also tests that we wrap back to the first server
+   * if we've tried them all.
+   */
+  @Test
+  public void testFailover() throws Exception {
+    Queue<String> ldapUrls = new LinkedList<>();
+    ldapUrls.add("ldap://test");
+    ldapUrls.add("ldap://test1");
+    ldapUrls.add("ldap://test2");
+
+    final int numAttempts = 12;
+    final int numAttemptsBeforeFailover = 2;
+
+    Configuration conf = getBaseConf();
+    conf.setStrings(LDAP_URL_KEY, "ldap://test", "ldap://test1",
+        "ldap://test2");
+    conf.setInt(LDAP_NUM_ATTEMPTS_KEY, numAttempts);
+    conf.setInt(LDAP_NUM_ATTEMPTS_BEFORE_FAILOVER_KEY,
+        numAttemptsBeforeFailover);
+
+    // Set the first expected url and add it back to the queue
+    String nextLdapUrl = ldapUrls.remove();
+    DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+    ldapUrls.add(nextLdapUrl);
+
+    // Number of attempts using a single ldap server url
+    final AtomicInteger serverAttempts = new AtomicInteger(
+        numAttemptsBeforeFailover);
+
+    when(getContext().search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class))).thenAnswer(new Answer<Object>() {
+          @Override
+          public Object answer(InvocationOnMock invocationOnMock)
+              throws Throwable {
+            if (serverAttempts.get() == 1) {
+              // Take the head of the queue and re-queue it to the back
+              String nextLdapUrl = ldapUrls.remove();
+              ldapUrls.add(nextLdapUrl);
+
+              DummyLdapCtxFactory.setExpectedLdapUrl(nextLdapUrl);
+              serverAttempts.set(numAttemptsBeforeFailover);
+            } else {
+              serverAttempts.decrementAndGet();
+            }
+            throw new CommunicationException();
+          }
+        });
+
+    LdapGroupsMapping groupsMapping = getGroupsMapping();
+    groupsMapping.setConf(conf);
+
+    List<String> groups = groupsMapping.getGroups(TEST_USER_NAME);
+    assertTrue(groups.isEmpty());
+
+    // Test that we made 6 attempts overall
+    verify(getContext(), times(numAttempts)).search(anyString(),
+        anyString(), any(Object[].class), any(SearchControls.class));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
index e5cd2b6..f51a944 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithOneQuery.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.security;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -68,7 +67,7 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // given a user whose ldap query returns a user object with three "memberOf"
     // properties, return an array of strings representing its groups.
     String[] testGroups = new String[] {"abc", "xyz", "sss"};
@@ -76,10 +75,9 @@ public class TestLdapGroupsMappingWithOneQuery
   }
 
   private void doTestGetGroups(List<String> expectedGroups)
-      throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      throws NamingException {
+    String ldapUrl = "ldap://test";
+    Configuration conf = getBaseConf(ldapUrl);
     // enable single-query lookup
     conf.set(LdapGroupsMapping.MEMBEROF_ATTR_KEY, "memberOf");
 
@@ -92,9 +90,7 @@ public class TestLdapGroupsMappingWithOneQuery
     Assert.assertEquals(expectedGroups, groups);
 
     // We should have only made one query because single-query lookup is enabled
-    verify(getContext(), times(1)).search(anyString(),
-        anyString(),
-        any(Object[].class),
-        any(SearchControls.class));
+    verify(getContext(), times(1)).search(anyString(), anyString(),
+        any(Object[].class), any(SearchControls.class));
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f8c879d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
index 17d28a5..4a21345 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMappingWithPosixGroup.java
@@ -28,7 +28,6 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -62,7 +61,7 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   @Test
-  public void testGetGroups() throws IOException, NamingException {
+  public void testGetGroups() throws NamingException {
     // The search functionality of the mock context is reused, so we will
     // return the user NamingEnumeration first, and then the group
     when(getContext().search(anyString(), contains("posix"),
@@ -73,10 +72,9 @@ public class TestLdapGroupsMappingWithPosixGroup
   }
 
   private void doTestGetGroups(List<String> expectedGroups, int searchTimes)
-      throws IOException, NamingException {
-    Configuration conf = new Configuration();
-    // Set this, so we don't throw an exception
-    conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
+      throws NamingException {
+    String ldapUrl = "ldap://test";
+    Configuration conf = getBaseConf(ldapUrl);
     conf.set(LdapGroupsMapping.GROUP_SEARCH_FILTER_KEY,
         "(objectClass=posixGroup)(cn={0})");
     conf.set(LdapGroupsMapping.USER_SEARCH_FILTER_KEY,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[23/50] [abbrv] hadoop git commit: Try something

Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 013c227..fe320af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -379,7 +379,7 @@ public class TestRMAdminCLI {
         new String[] {"group1", "group2"});
     PrintStream origOut = System.out;
     PrintStream out = mock(PrintStream.class);
-    System.setOut(out);
+    //System.setOut(out);
     try {
       String[] args = { "-getGroups", "admin" };
       assertEquals(0, rmAdminCLI.run(args));
@@ -391,7 +391,7 @@ public class TestRMAdminCLI {
         }
       }));
     } finally {
-      System.setOut(origOut);
+      //System.setOut(origOut);
     }
   }
 
@@ -488,8 +488,8 @@ public class TestRMAdminCLI {
     PrintStream oldErrPrintStream = System.err;
     ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
     ByteArrayOutputStream dataErr = new ByteArrayOutputStream();
-    System.setOut(new PrintStream(dataOut));
-    System.setErr(new PrintStream(dataErr));
+    //System.setOut(new PrintStream(dataOut));
+    //System.setErr(new PrintStream(dataErr));
     try {
       String[] args = { "-help" };
       assertEquals(0, rmAdminCLI.run(args));
@@ -619,8 +619,8 @@ public class TestRMAdminCLI {
           "messages: %n" + expectedHelpMsg), actualHelpMsg.contains(expectedHelpMsg
               ));
     } finally {
-      System.setOut(oldOutPrintStream);
-      System.setErr(oldErrPrintStream);
+      //System.setOut(oldOutPrintStream);
+      //System.setErr(oldErrPrintStream);
     }
   }
 
@@ -628,7 +628,7 @@ public class TestRMAdminCLI {
   public void testException() throws Exception {
     PrintStream oldErrPrintStream = System.err;
     ByteArrayOutputStream dataErr = new ByteArrayOutputStream();
-    System.setErr(new PrintStream(dataErr));
+    //System.setErr(new PrintStream(dataErr));
     try {
       when(admin.refreshQueues(any(RefreshQueuesRequest.class)))
           .thenThrow(new IOException("test exception"));
@@ -638,7 +638,7 @@ public class TestRMAdminCLI {
       verify(admin).refreshQueues(any(RefreshQueuesRequest.class));
       assertTrue(dataErr.toString().contains("refreshQueues: test exception"));
     } finally {
-      System.setErr(oldErrPrintStream);
+      //System.setErr(oldErrPrintStream);
     }
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index 5364e83..eba71b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -49,11 +49,11 @@ public class TestSchedConfCLI {
   public void setUp() {
     sysOutStream = new ByteArrayOutputStream();
     sysOut =  new PrintStream(sysOutStream);
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
 
     sysErrStream = new ByteArrayOutputStream();
     sysErr = new PrintStream(sysErrStream);
-    System.setErr(sysErr);
+    //System.setErr(sysErr);
 
     cli = new SchedConfCLI();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 0f0c9bf..35b1f9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -110,7 +110,7 @@ public class TestYarnCLI {
     sysOut = spy(new PrintStream(sysOutStream));
     sysErrStream = new ByteArrayOutputStream();
     sysErr = spy(new PrintStream(sysErrStream));
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
   }
   
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
index 7d0205b..f789f86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/filecontroller/ifile/TestLogAggregationIndexFileController.java
@@ -104,11 +104,11 @@ public class TestLogAggregationIndexFileController {
     fs = FileSystem.get(conf);
     sysOutStream = new ByteArrayOutputStream();
     sysOut =  new PrintStream(sysOutStream);
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
 
     sysErrStream = new ByteArrayOutputStream();
     sysErr = new PrintStream(sysErrStream);
-    System.setErr(sysErr);
+    //System.setErr(sysErr);
   }
 
   @After

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b580aaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
index bd8a38d..086a933 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
@@ -44,7 +44,7 @@ public class TestRegistryCli extends AbstractRegistryTest {
     sysOut = new PrintStream(sysOutStream);
     sysErrStream = new ByteArrayOutputStream();
     sysErr = new PrintStream(sysErrStream);
-    System.setOut(sysOut);
+    //System.setOut(sysOut);
     cli = new RegistryCli(operations, createRegistryConfiguration(), sysOut, sysErr);
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[49/50] [abbrv] hadoop git commit: HADOOP-15917. AliyunOSS: fix incorrect ReadOps and WriteOps in statistics. Contributed by Jinhu Wu.

Posted by jh...@apache.org.
HADOOP-15917. AliyunOSS: fix incorrect ReadOps and WriteOps in statistics. Contributed by Jinhu Wu.

(cherry picked from commit 3fade865ce84dcf68bcd7de5a5ed1c7d904796e9)
(cherry picked from commit 64cb97fb4467513f73fde18f96f391ad34e3bb0a)
(cherry picked from commit 5d532cfc6f23f942ed10edab55ed251eb99a0664)
(cherry picked from commit 37082a664aaf99bc40522a8dfa231d71792dd976)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0f9757fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0f9757fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0f9757fb

Branch: refs/heads/branch-2-jhung-test
Commit: 0f9757fb717680ca37067f76487e99e6c000cd39
Parents: f64b451
Author: Sammi Chen <sa...@intel.com>
Authored: Wed Nov 14 12:58:57 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      |  4 --
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 22 ++++--
 .../site/markdown/tools/hadoop-aliyun/index.md  |  5 ++
 .../oss/TestAliyunOSSBlockOutputStream.java     | 70 +++++++++++++++++---
 4 files changed, 83 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f9757fb/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 7356818..809c8c8 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -457,7 +457,6 @@ public class AliyunOSSFileSystem extends FileSystem {
 
       ObjectListing objects = store.listObjects(key, maxKeys, null, false);
       while (true) {
-        statistics.incrementReadOps(1);
         for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
           String objKey = objectSummary.getKey();
           if (objKey.equals(key + "/")) {
@@ -498,7 +497,6 @@ public class AliyunOSSFileSystem extends FileSystem {
           }
           String nextMarker = objects.getNextMarker();
           objects = store.listObjects(key, maxKeys, nextMarker, false);
-          statistics.incrementReadOps(1);
         } else {
           break;
         }
@@ -694,7 +692,6 @@ public class AliyunOSSFileSystem extends FileSystem {
         new SemaphoredDelegatingExecutor(boundedCopyThreadPool,
             maxConcurrentCopyTasksPerDir, true));
     ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true);
-    statistics.incrementReadOps(1);
     // Copy files from src folder to dst
     int copiesToFinish = 0;
     while (true) {
@@ -717,7 +714,6 @@ public class AliyunOSSFileSystem extends FileSystem {
       if (objects.isTruncated()) {
         String nextMarker = objects.getNextMarker();
         objects = store.listObjects(srcKey, maxKeys, nextMarker, true);
-        statistics.incrementReadOps(1);
       } else {
         break;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f9757fb/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index f13ac32..f0413e3 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -175,6 +175,7 @@ public class AliyunOSSFileSystemStore {
       CannedAccessControlList cannedACL =
           CannedAccessControlList.valueOf(cannedACLName);
       ossClient.setBucketAcl(bucketName, cannedACL);
+      statistics.incrementWriteOps(1);
     }
 
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
@@ -216,6 +217,7 @@ public class AliyunOSSFileSystemStore {
       // Here, we choose the simple mode to do batch delete.
       deleteRequest.setQuiet(true);
       DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest);
+      statistics.incrementWriteOps(1);
       deleteFailed = result.getDeletedObjects();
       tries++;
       if (tries == retry) {
@@ -268,11 +270,13 @@ public class AliyunOSSFileSystemStore {
    */
   public ObjectMetadata getObjectMetadata(String key) {
     try {
-      return ossClient.getObjectMetadata(bucketName, key);
+      ObjectMetadata objectMeta = ossClient.getObjectMetadata(bucketName, key);
+      statistics.incrementReadOps(1);
+      return objectMeta;
     } catch (OSSException osse) {
+      LOG.error("Exception thrown when get object meta: "
+              + key + ", exception: " + osse);
       return null;
-    } finally {
-      statistics.incrementReadOps(1);
     }
   }
 
@@ -289,6 +293,7 @@ public class AliyunOSSFileSystemStore {
     dirMeta.setContentLength(0);
     try {
       ossClient.putObject(bucketName, key, in, dirMeta);
+      statistics.incrementWriteOps(1);
     } finally {
       in.close();
     }
@@ -304,6 +309,7 @@ public class AliyunOSSFileSystemStore {
   public boolean copyFile(String srcKey, String dstKey) {
     ObjectMetadata objectMeta =
         ossClient.getObjectMetadata(bucketName, srcKey);
+    statistics.incrementReadOps(1);
     long contentLength = objectMeta.getContentLength();
     if (contentLength <= multipartThreshold) {
       return singleCopy(srcKey, dstKey);
@@ -323,6 +329,7 @@ public class AliyunOSSFileSystemStore {
   private boolean singleCopy(String srcKey, String dstKey) {
     CopyObjectResult copyResult =
         ossClient.copyObject(bucketName, srcKey, bucketName, dstKey);
+    statistics.incrementWriteOps(1);
     LOG.debug(copyResult.getETag());
     return true;
   }
@@ -372,6 +379,7 @@ public class AliyunOSSFileSystemStore {
         UploadPartCopyResult partCopyResult =
             ossClient.uploadPartCopy(partCopyRequest);
         statistics.incrementWriteOps(1);
+        statistics.incrementBytesWritten(size);
         partETags.add(partCopyResult.getPartETag());
       }
       CompleteMultipartUploadRequest completeMultipartUploadRequest =
@@ -408,6 +416,7 @@ public class AliyunOSSFileSystemStore {
       PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
       LOG.debug(result.getETag());
       statistics.incrementWriteOps(1);
+      statistics.incrementBytesWritten(file.length());
     } finally {
       fis.close();
     }
@@ -449,7 +458,9 @@ public class AliyunOSSFileSystemStore {
     try {
       GetObjectRequest request = new GetObjectRequest(bucketName, key);
       request.setRange(byteStart, byteEnd);
-      return ossClient.getObject(request).getObjectContent();
+      InputStream in = ossClient.getObject(request).getObjectContent();
+      statistics.incrementReadOps(1);
+      return in;
     } catch (OSSException | ClientException e) {
       LOG.error("Exception thrown when store retrieves key: "
               + key + ", exception: " + e);
@@ -480,6 +491,7 @@ public class AliyunOSSFileSystemStore {
       for (OSSObjectSummary object : objects.getObjectSummaries()) {
         key = object.getKey();
         ossClient.deleteObject(bucketName, key);
+        statistics.incrementWriteOps(1);
       }
 
       for (String dir: objects.getCommonPrefixes()) {
@@ -605,6 +617,8 @@ public class AliyunOSSFileSystemStore {
         uploadRequest.setPartSize(file.length());
         uploadRequest.setPartNumber(idx);
         UploadPartResult uploadResult = ossClient.uploadPart(uploadRequest);
+        statistics.incrementWriteOps(1);
+        statistics.incrementBytesWritten(file.length());
         return uploadResult.getPartETag();
       } catch (Exception e) {
         LOG.debug("Failed to upload "+ file.getPath() +", " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f9757fb/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 0c3131d..87aa90b 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -118,6 +118,11 @@ please raise your issues with them.
     </property>
 
     <property>
+    <name>fs.oss.impl</name>
+    <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
+    </property>
+
+    <property>
       <name>fs.oss.assumed.role.arn</name>
       <description>
         Role ARN for the role to be assumed.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f9757fb/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
index 6fe6f03..c3387a3 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
@@ -32,6 +32,7 @@ import java.io.IOException;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.IO_CHUNK_BUFFER_SIZE;
+import static org.junit.Assert.assertEquals;
 
 /**
  * Tests regular and multi-part upload functionality for
@@ -74,24 +75,73 @@ public class TestAliyunOSSBlockOutputStream {
 
   @Test
   public void testRegularUpload() throws IOException {
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 - 1);
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 + 1);
+    long size = 1024 * 1024;
+    FileSystem.Statistics statistics =
+        FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
+    // This test is a little complicated for statistics, lifecycle is
+    // generateTestFile
+    //   fs.create(getFileStatus)    read 1
+    //   output stream write         write 1
+    // path exists(fs.exists)        read 1
+    // verifyReceivedData
+    //   fs.open(getFileStatus)      read 1
+    //   input stream read           read 2(part size is 512K)
+    // fs.delete
+    //   getFileStatus & delete & exists & create fake dir read 2, write 2
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size - 1);
+    assertEquals(7, statistics.getReadOps());
+    assertEquals(size - 1, statistics.getBytesRead());
+    assertEquals(3, statistics.getWriteOps());
+    assertEquals(size - 1, statistics.getBytesWritten());
+
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size);
+    assertEquals(14, statistics.getReadOps());
+    assertEquals(2 * size - 1, statistics.getBytesRead());
+    assertEquals(6, statistics.getWriteOps());
+    assertEquals(2 * size - 1, statistics.getBytesWritten());
+
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size + 1);
+
+    assertEquals(22, statistics.getReadOps());
+    assertEquals(3 * size, statistics.getBytesRead());
+    assertEquals(10, statistics.getWriteOps());
+    assertEquals(3 * size, statistics.getBytesWritten());
   }
 
   @Test
   public void testMultiPartUpload() throws IOException {
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
-        6 * 1024 * 1024 - 1);
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
-        6 * 1024 * 1024 + 1);
+    long size = 6 * 1024 * 1024;
+    FileSystem.Statistics statistics =
+        FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size - 1);
+    assertEquals(17, statistics.getReadOps());
+    assertEquals(size - 1, statistics.getBytesRead());
+    assertEquals(8, statistics.getWriteOps());
+    assertEquals(size - 1, statistics.getBytesWritten());
+
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size);
+    assertEquals(34, statistics.getReadOps());
+    assertEquals(2 * size - 1, statistics.getBytesRead());
+    assertEquals(16, statistics.getWriteOps());
+    assertEquals(2 * size - 1, statistics.getBytesWritten());
+
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size + 1);
+    assertEquals(52, statistics.getReadOps());
+    assertEquals(3 * size, statistics.getBytesRead());
+    assertEquals(25, statistics.getWriteOps());
+    assertEquals(3 * size, statistics.getBytesWritten());
   }
 
   @Test
   public void testMultiPartUploadConcurrent() throws IOException {
-    ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
-        50 * 1024 * 1024 - 1);
+    long size = 50 * 1024 * 1024 - 1;
+    ContractTestUtils.createAndVerifyFile(fs, getTestPath(), size);
+    FileSystem.Statistics statistics =
+        FileSystem.getStatistics("oss", AliyunOSSFileSystem.class);
+    assertEquals(105, statistics.getReadOps());
+    assertEquals(size, statistics.getBytesRead());
+    assertEquals(52, statistics.getWriteOps());
+    assertEquals(size, statistics.getBytesWritten());
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[41/50] [abbrv] hadoop git commit: HDFS-14043. Tolerate corrupted seen_txid file. Contributed by Lukas Majercak.

Posted by jh...@apache.org.
HDFS-14043. Tolerate corrupted seen_txid file. Contributed by Lukas Majercak.

(cherry picked from commit f3296501e09fa7f1e81548dfcefa56f20fe337ca)
(cherry picked from commit 9bf4f3d61403b06f3d6a092dacab382c7c131e19)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4044fda
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4044fda
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4044fda

Branch: refs/heads/branch-2-jhung-test
Commit: a4044fda15b34d7df7ff2160bb9ecf400627deac
Parents: f87ba60
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Nov 5 16:48:37 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop/hdfs/util/PersistentLongFile.java    |  2 +
 .../hdfs/server/namenode/TestSaveNamespace.java | 56 ++++++++++++++++++++
 2 files changed, 58 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4044fda/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
index 67bb2bb..b2075cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/PersistentLongFile.java
@@ -98,6 +98,8 @@ public class PersistentLongFile {
         val = Long.parseLong(br.readLine());
         br.close();
         br = null;
+      } catch (NumberFormatException e) {
+        throw new IOException(e);
       } finally {
         IOUtils.cleanup(LOG, br);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4044fda/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index 5050998..1f201f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -28,8 +28,13 @@ import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.spy;
 
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -37,6 +42,8 @@ import java.util.concurrent.Future;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -687,6 +694,55 @@ public class TestSaveNamespace {
     }
   }
 
+  @Test(timeout=30000)
+  public void testTxFaultTolerance() throws Exception {
+    String baseDir = MiniDFSCluster.getBaseDirectory();
+    List<String> nameDirs = new ArrayList<>();
+    nameDirs.add(fileAsURI(new File(baseDir, "name1")).toString());
+    nameDirs.add(fileAsURI(new File(baseDir, "name2")).toString());
+
+    Configuration conf = new HdfsConfiguration();
+    String nameDirsStr = StringUtils.join(",", nameDirs);
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, nameDirsStr);
+    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, nameDirsStr);
+
+    NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
+    DFSTestUtil.formatNameNode(conf);
+    FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
+    try {
+      // We have a BEGIN_LOG_SEGMENT txn to start
+      assertEquals(1, fsn.getEditLog().getLastWrittenTxId());
+
+      doAnEdit(fsn, 1);
+
+      assertEquals(2, fsn.getEditLog().getLastWrittenTxId());
+
+      // Shut down
+      fsn.close();
+
+      // Corrupt one of the seen_txid files
+      File txidFile0 = new File(new URI(nameDirs.get(0) +
+          "/current/seen_txid"));
+      FileWriter fw = new FileWriter(txidFile0, false);
+      try (PrintWriter pw = new PrintWriter(fw)) {
+        pw.print("corrupt____!");
+      }
+
+      // Restart
+      fsn = FSNamesystem.loadFromDisk(conf);
+      assertEquals(4, fsn.getEditLog().getLastWrittenTxId());
+
+      // Check seen_txid is same in both dirs
+      File txidFile1 = new File(new URI(nameDirs.get(1) +
+          "/current/seen_txid"));
+      assertTrue(FileUtils.contentEquals(txidFile0, txidFile1));
+    } finally {
+      if (fsn != null) {
+        fsn.close();
+      }
+    }
+  }
+
   private void doAnEdit(FSNamesystem fsn, int id) throws IOException {
     // Make an edit
     fsn.mkdirs(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[02/50] [abbrv] hadoop git commit: YARN-8905. [Router] Add JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.

Posted by jh...@apache.org.
YARN-8905. [Router] Add JvmMetricsInfo and pause monitor. Contributed by Bilwa S T.

(cherry picked from commit f84a278baad6cb871ea4196257f23a938826ca23)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85c2143c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85c2143c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85c2143c

Branch: refs/heads/branch-2-jhung-test
Commit: 85c2143c8a6efa631aafcab764243f2b213c3223
Parents: 690d808
Author: bibinchundatt <bi...@apache.org>
Authored: Sat Nov 3 20:35:31 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/server/router/Router.java       |  8 +++++
 .../hadoop/yarn/server/router/TestRouter.java   | 38 ++++++++++++++++++++
 2 files changed, 46 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/85c2143c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
index 76050d0..b55c5d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
@@ -24,7 +24,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
 import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.JvmPauseMonitor;
 import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
@@ -64,6 +66,7 @@ public class Router extends CompositeService {
   private static CompositeServiceShutdownHook routerShutdownHook;
   private Configuration conf;
   private AtomicBoolean isStopping = new AtomicBoolean(false);
+  private JvmPauseMonitor pauseMonitor;
   private RouterClientRMService clientRMProxyService;
   private RouterRMAdminService rmAdminProxyService;
   private WebApp webApp;
@@ -100,6 +103,11 @@ public class Router extends CompositeService {
         WebAppUtils.getRouterWebAppURLWithoutScheme(this.conf));
     // Metrics
     DefaultMetricsSystem.initialize(METRICS_NAME);
+    JvmMetrics jm = JvmMetrics.initSingleton("Router", null);
+    pauseMonitor = new JvmPauseMonitor();
+    addService(pauseMonitor);
+    jm.setPauseMonitor(pauseMonitor);
+
     super.serviceInit(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85c2143c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
new file mode 100644
index 0000000..bf0c688
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+/**
+ * Tests {@link Router}.
+ */
+public class TestRouter {
+
+  @Test
+  public void testJVMMetricsService() {
+    YarnConfiguration conf = new YarnConfiguration();
+    Router router = new Router();
+    router.init(conf);
+    assertEquals(3, router.getServices().size());
+  }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[13/50] [abbrv] hadoop git commit: YARN-7274. Ability to disable elasticity at leaf queue level. (Zian Chen via wangda)

Posted by jh...@apache.org.
YARN-7274. Ability to disable elasticity at leaf queue level. (Zian Chen via wangda)

Change-Id: Ic8d43e297f0f5de788b562f7eff8106c5c35e8d2
(cherry picked from commit 74665e3a7d7f05644d9a5abad5a3f2d47597d6c8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1045b699
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1045b699
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1045b699

Branch: refs/heads/branch-2-jhung-test
Commit: 1045b69982d887b306ba57b1d4104043682b4bd8
Parents: 079aedb
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 15:07:56 2017 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../scheduler/capacity/CSQueueUtils.java        |  16 +-
 .../TestNodeLabelContainerAllocation.java       | 163 ++++++++++++++++++
 .../scheduler/capacity/TestQueueParsing.java    | 168 ++++++++++++++++++-
 3 files changed, 329 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1045b699/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e1014c1..9554086 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -66,19 +66,9 @@ class CSQueueUtils {
   private static void capacitiesSanityCheck(String queueName,
       QueueCapacities queueCapacities) {
     for (String label : queueCapacities.getExistingNodeLabels()) {
-      float capacity = queueCapacities.getCapacity(label);
-      float maximumCapacity = queueCapacities.getMaximumCapacity(label);
-      if (capacity > maximumCapacity) {
-        throw new IllegalArgumentException("Illegal queue capacity setting, "
-            + "(capacity=" + capacity + ") > (maximum-capacity="
-            + maximumCapacity + "). When label=[" + label + "]");
-      }
-     
-      // Actually, this may not needed since we have verified capacity <=
-      // maximumCapacity. And the way we compute absolute capacity (abs(x) =
-      // cap(x) * cap(x.parent) * ...) is a monotone increasing function. But
-      // just keep it here to make sure our compute abs capacity method works
-      // correctly. 
+      // The only thing we should care about is absolute capacity <=
+      // absolute max capacity otherwise the absolute max capacity is
+      // no longer an absolute maximum.
       float absCapacity = queueCapacities.getAbsoluteCapacity(label);
       float absMaxCapacity = queueCapacities.getAbsoluteMaximumCapacity(label);
       if (absCapacity > absMaxCapacity) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1045b699/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 17c5797..4c0ca6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -2145,4 +2145,167 @@ public class TestNodeLabelContainerAllocation {
 
     rm1.close();
   }
+
+  @Test
+  public void testQueueMetricsWithLabelsDisableElasticity() throws Exception {
+    /**
+     * Test case: have a following queue structure:
+     *
+     * <pre>
+     *
+     *          root
+     *        /      \
+     *       a        b
+     *      (x)      (x)
+     *      / \
+     *     a1 a2
+     *    (x) (x)
+     * </pre>
+     *
+     * a/b can access x, both of them has max-capacity-on-x = 50
+     *
+     * When doing non-exclusive allocation, app in a (or b) can use 100% of x
+     * resource.
+     */
+
+    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
+        this.conf);
+
+    // Define top-level queues
+    csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+        new String[] { "a", "b" });
+    csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+
+    final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
+    csConf.setCapacity(queueA, 50);
+    csConf.setMaximumCapacity(queueA, 100);
+    csConf.setAccessibleNodeLabels(queueA, toSet("x"));
+    csConf.setCapacityByLabel(queueA, "x", 50);
+    csConf.setMaximumCapacityByLabel(queueA, "x", 100);
+    final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
+    csConf.setCapacity(queueB, 50);
+    csConf.setMaximumCapacity(queueB, 100);
+    csConf.setAccessibleNodeLabels(queueB, toSet("x"));
+    csConf.setCapacityByLabel(queueB, "x", 50);
+    csConf.setMaximumCapacityByLabel(queueB, "x", 100);
+
+    // Define 2nd-level queues
+    csConf.setQueues(queueA, new String[] { "a1",
+        "a2"});
+
+    final String A1 = queueA + ".a1";
+    csConf.setCapacity(A1, 20);
+    csConf.setMaximumCapacity(A1, 60);
+    csConf.setAccessibleNodeLabels(A1, toSet("x"));
+    csConf.setCapacityByLabel(A1, "x", 60);
+    csConf.setMaximumCapacityByLabel(A1, "x", 30);
+
+    final String A2 = queueA + ".a2";
+    csConf.setCapacity(A2, 80);
+    csConf.setMaximumCapacity(A2, 40);
+    csConf.setAccessibleNodeLabels(A2, toSet("x"));
+    csConf.setCapacityByLabel(A2, "x", 40);
+    csConf.setMaximumCapacityByLabel(A2, "x", 20);
+
+    // set node -> label
+    mgr.addToCluserNodeLabels(
+        ImmutableSet.of(NodeLabel.newInstance("x", false)));
+    mgr.addLabelsToNode(
+        ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+
+    // inject node label manager
+    MockRM rm1 = new MockRM(csConf) {
+      @Override
+      public RMNodeLabelsManager createNodeLabelManager() {
+        return mgr;
+      }
+    };
+
+    rm1.getRMContext().setNodeLabelManager(mgr);
+    rm1.start();
+    MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB); // label = x
+    // app1 -> a1
+    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+    // app1 asks for 6 partition=x containers
+    am1.allocate("*", 1 * GB, 6, new ArrayList<ContainerId>(), "x");
+
+    // NM1 do 50 heartbeats
+    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
+        cs.getApplicationAttempt(am1.getApplicationAttemptId()));
+
+    SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
+        .getNodeReport(nm1.getNodeId());
+    Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(14 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Try to launch app2 in a2, asked 2GB, should success
+    // app2 -> a2
+    RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x");
+    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+    // app2 asks for 4 partition=x containers
+    am2.allocate("*", 1 * GB, 4, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
+        cs.getApplicationAttempt(am2.getApplicationAttemptId()));
+
+    reportNm1 = rm1.getResourceScheduler()
+        .getNodeReport(nm1.getNodeId());
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Kill all apps in queue a2
+    cs.killAllAppsInQueue("a2");
+    rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
+    rm1.waitForAppRemovedFromScheduler(app2.getApplicationId());
+
+    // Try to launch app3 in a2, asked 6GB, should fail
+    // app3 -> a2
+    RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a2", "x");
+    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
+
+    am3.allocate("*", 1 * GB, 6, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+    // app3 cannot preempt more resources restricted by disable elasticity
+    checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
+        cs.getApplicationAttempt(am3.getApplicationAttemptId()));
+
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    // Kill all apps in queue a1
+    cs.killAllAppsInQueue("a1");
+    rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
+    rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
+
+    // app4 -> a1, try to allocate more than 6GB resource, should fail
+    RMApp app4 = rm1.submitApp(1 * GB, "app", "user", null, "a1", "x");
+    MockAM am4 = MockRM.launchAndRegisterAM(app4, rm1, nm1);
+
+    // app3 asks for 7 partition=x containers
+    am4.allocate("*", 1 * GB, 7, new ArrayList<ContainerId>(), "x");
+    // NM1 do 50 heartbeats
+    doNMHeartbeat(rm1, nm1.getNodeId(), 50);
+
+    // app4 should only gets 6GB resource in partition=x
+    // since elasticity is disabled
+    checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
+        cs.getApplicationAttempt(am4.getApplicationAttemptId()));
+
+    Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
+    Assert.assertEquals(10 * GB,
+        reportNm1.getAvailableResource().getMemorySize());
+
+    rm1.close();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1045b699/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 5d167c7..add14ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -365,7 +365,76 @@ public class TestQueueParsing {
     conf.setCapacityByLabel(B3, "red", 25);
     conf.setCapacityByLabel(B3, "blue", 25);
   }
-  
+
+  private void setupQueueConfigurationWithLabelsAndReleaseCheck
+      (CapacitySchedulerConfiguration conf) {
+    // Define top-level queues
+    conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100);
+    conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100);
+
+    final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+    // The cap <= max-cap check is not needed
+    conf.setCapacity(A, 50);
+    conf.setMaximumCapacity(A, 100);
+
+    final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+    conf.setCapacity(B, 50);
+    conf.setMaximumCapacity(B, 100);
+
+    LOG.info("Setup top-level queues");
+
+    // Define 2nd-level queues
+    final String A1 = A + ".a1";
+    final String A2 = A + ".a2";
+    conf.setQueues(A, new String[] {"a1", "a2"});
+    conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue"));
+    conf.setCapacityByLabel(A, "red", 50);
+    conf.setMaximumCapacityByLabel(A, "red", 100);
+    conf.setCapacityByLabel(A, "blue", 30);
+    conf.setMaximumCapacityByLabel(A, "blue", 50);
+
+    conf.setCapacity(A1, 60);
+    conf.setMaximumCapacity(A1, 60);
+    conf.setCapacityByLabel(A1, "red", 60);
+    conf.setMaximumCapacityByLabel(A1, "red", 30);
+    conf.setCapacityByLabel(A1, "blue", 100);
+    conf.setMaximumCapacityByLabel(A1, "blue", 100);
+
+    conf.setCapacity(A2, 40);
+    conf.setMaximumCapacity(A2, 85);
+    conf.setAccessibleNodeLabels(A2, ImmutableSet.of("red"));
+    conf.setCapacityByLabel(A2, "red", 40);
+    conf.setMaximumCapacityByLabel(A2, "red", 60);
+
+    final String B1 = B + ".b1";
+    final String B2 = B + ".b2";
+    final String B3 = B + ".b3";
+    conf.setQueues(B, new String[] {"b1", "b2", "b3"});
+    conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue"));
+    conf.setCapacityByLabel(B, "red", 50);
+    conf.setMaximumCapacityByLabel(B, "red", 100);
+    conf.setCapacityByLabel(B, "blue", 70);
+    conf.setMaximumCapacityByLabel(B, "blue", 100);
+
+    conf.setCapacity(B1, 10);
+    conf.setMaximumCapacity(B1, 10);
+    conf.setCapacityByLabel(B1, "red", 60);
+    conf.setMaximumCapacityByLabel(B1, "red", 30);
+    conf.setCapacityByLabel(B1, "blue", 50);
+    conf.setMaximumCapacityByLabel(B1, "blue", 100);
+
+    conf.setCapacity(B2, 80);
+    conf.setMaximumCapacity(B2, 40);
+    conf.setCapacityByLabel(B2, "red", 30);
+    conf.setCapacityByLabel(B2, "blue", 25);
+
+    conf.setCapacity(B3, 10);
+    conf.setMaximumCapacity(B3, 25);
+    conf.setCapacityByLabel(B3, "red", 10);
+    conf.setCapacityByLabel(B3, "blue", 25);
+  }
+
   private void setupQueueConfigurationWithLabelsInherit(
       CapacitySchedulerConfiguration conf) {
     // Define top-level queues
@@ -472,7 +541,7 @@ public class TestQueueParsing {
     // queue-B2 inherits "red"/"blue"
     Assert.assertTrue(capacityScheduler.getQueue("b2")
         .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
-    
+
     // check capacity of A2
     CSQueue qA2 = capacityScheduler.getQueue("a2");
     Assert.assertEquals(0.7, qA2.getCapacity(), DELTA);
@@ -481,7 +550,7 @@ public class TestQueueParsing {
     Assert.assertEquals(0.25, qA2.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
     Assert.assertEquals(0.1275, qA2.getAbsoluteMaximumCapacity(), DELTA);
     Assert.assertEquals(0.3, qA2.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
-    
+
     // check capacity of B3
     CSQueue qB3 = capacityScheduler.getQueue("b3");
     Assert.assertEquals(0.18, qB3.getAbsoluteCapacity(), DELTA);
@@ -489,7 +558,71 @@ public class TestQueueParsing {
     Assert.assertEquals(0.35, qB3.getAbsoluteMaximumCapacity(), DELTA);
     Assert.assertEquals(1, qB3.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
   }
-  
+
+  private void checkQueueLabelsWithLeafQueueDisableElasticity
+      (CapacityScheduler capacityScheduler) {
+    // queue-A is red, blue
+    Assert.assertTrue(capacityScheduler.getQueue("a").getAccessibleNodeLabels()
+        .containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-A1 inherits A's configuration
+    Assert.assertTrue(capacityScheduler.getQueue("a1")
+        .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-A2 is "red"
+    Assert.assertEquals(1, capacityScheduler.getQueue("a2")
+        .getAccessibleNodeLabels().size());
+    Assert.assertTrue(capacityScheduler.getQueue("a2")
+        .getAccessibleNodeLabels().contains("red"));
+
+    // queue-B is "red"/"blue"
+    Assert.assertTrue(capacityScheduler.getQueue("b").getAccessibleNodeLabels()
+        .containsAll(ImmutableSet.of("red", "blue")));
+
+    // queue-B2 inherits "red"/"blue"
+    Assert.assertTrue(capacityScheduler.getQueue("b2")
+        .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue")));
+
+    // check capacity of A2
+    CSQueue qA2 = capacityScheduler.getQueue("a2");
+    Assert.assertEquals(0.4, qA2.getCapacity(), DELTA);
+    Assert.assertEquals(0.4, qA2.getQueueCapacities()
+        .getCapacity("red"), DELTA);
+    Assert.assertEquals(0.2, qA2.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.2, qA2.getQueueCapacities()
+        .getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.85, qA2.getAbsoluteMaximumCapacity(), DELTA);
+    Assert.assertEquals(0.6, qA2.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("red"), DELTA);
+
+    // check disable elasticity at leaf queue level without label
+    CSQueue qB2 = capacityScheduler.getQueue("b2");
+    Assert.assertEquals(0.4, qB2.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.4, qB2.getAbsoluteMaximumCapacity(), DELTA);
+
+    // check disable elasticity at leaf queue level with label
+    CSQueue qA1 = capacityScheduler.getQueue("a1");
+    Assert.assertEquals(0.3, qA1.getQueueCapacities().
+        getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.3, qA1.getQueueCapacities().
+        getAbsoluteMaximumCapacity("red"), DELTA);
+
+    CSQueue qB1 = capacityScheduler.getQueue("b1");
+    Assert.assertEquals(0.3, qB1.getQueueCapacities()
+        .getAbsoluteCapacity("red"), DELTA);
+    Assert.assertEquals(0.3, qB1.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("red"), DELTA);
+
+    // check capacity of B3
+    CSQueue qB3 = capacityScheduler.getQueue("b3");
+    Assert.assertEquals(0.05, qB3.getAbsoluteCapacity(), DELTA);
+    Assert.assertEquals(0.175, qB3.getQueueCapacities()
+        .getAbsoluteCapacity("blue"), DELTA);
+    Assert.assertEquals(0.25, qB3.getAbsoluteMaximumCapacity(), DELTA);
+    Assert.assertEquals(1, qB3.getQueueCapacities()
+        .getAbsoluteMaximumCapacity("blue"), DELTA);
+  }
+
   private void
       checkQueueLabelsInheritConfig(CapacityScheduler capacityScheduler) {
     // queue-A is red, blue
@@ -514,7 +647,7 @@ public class TestQueueParsing {
   @Test
   public void testQueueParsingWithLabels() throws IOException {
     nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue"));
-    
+
     YarnConfiguration conf = new YarnConfiguration();
     CapacitySchedulerConfiguration csConf =
         new CapacitySchedulerConfiguration(conf);
@@ -534,6 +667,31 @@ public class TestQueueParsing {
     checkQueueLabels(capacityScheduler);
     ServiceOperations.stopQuietly(capacityScheduler);
   }
+
+  @Test
+  public void testQueueParsingWithLeafQueueDisableElasticity()
+      throws IOException {
+    nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity
+        (ImmutableSet.of("red", "blue"));
+
+    YarnConfiguration conf = new YarnConfiguration();
+    CapacitySchedulerConfiguration csConf =
+        new CapacitySchedulerConfiguration(conf);
+    setupQueueConfigurationWithLabelsAndReleaseCheck(csConf);
+    CapacityScheduler capacityScheduler = new CapacityScheduler();
+    RMContextImpl rmContext =
+        new RMContextImpl(null, null, null, null, null, null,
+            new RMContainerTokenSecretManager(csConf),
+            new NMTokenSecretManagerInRM(csConf),
+            new ClientToAMTokenSecretManagerInRM(), null);
+    rmContext.setNodeLabelManager(nodeLabelManager);
+    capacityScheduler.setConf(csConf);
+    capacityScheduler.setRMContext(rmContext);
+    capacityScheduler.init(csConf);
+    capacityScheduler.start();
+    checkQueueLabelsWithLeafQueueDisableElasticity(capacityScheduler);
+    ServiceOperations.stopQuietly(capacityScheduler);
+  }
   
   @Test
   public void testQueueParsingWithLabelsInherit() throws IOException {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[30/50] [abbrv] hadoop git commit: YARN-8933. [AMRMProxy] Fix potential empty fields in allocation response, move SubClusterTimeout to FederationInterceptor. Contributed by Botong Huang.

Posted by jh...@apache.org.
YARN-8933. [AMRMProxy] Fix potential empty fields in allocation response, move SubClusterTimeout to FederationInterceptor. Contributed by Botong Huang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00b4455d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00b4455d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00b4455d

Branch: refs/heads/branch-2-jhung-test
Commit: 00b4455d3c9ee284d960fadb8c4d9da647218c2a
Parents: 181e138
Author: Botong Huang <bo...@apache.org>
Authored: Sun Nov 11 11:12:53 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../amrmproxy/BroadcastAMRMProxyPolicy.java     |   4 +-
 .../amrmproxy/FederationAMRMProxyPolicy.java    |   8 +-
 .../LocalityMulticastAMRMProxyPolicy.java       |  52 ++------
 .../amrmproxy/RejectAMRMProxyPolicy.java        |   4 +-
 .../policies/BaseFederationPoliciesTest.java    |   5 +-
 .../TestBroadcastAMRMProxyFederationPolicy.java |  10 +-
 .../TestLocalityMulticastAMRMProxyPolicy.java   |  41 +++---
 .../amrmproxy/TestRejectAMRMProxyPolicy.java    |   5 +-
 .../amrmproxy/FederationInterceptor.java        | 129 ++++++++++++++++---
 .../amrmproxy/TestFederationInterceptor.java    |  63 ++++++++-
 10 files changed, 227 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
index 7fddb8e..ec8f743 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -50,7 +51,8 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
 
   @Override
   public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
-      List<ResourceRequest> resourceRequests) throws YarnException {
+      List<ResourceRequest> resourceRequests,
+      Set<SubClusterId> timedOutSubClusters) throws YarnException {
 
     Map<SubClusterId, SubClusterInfo> activeSubclusters =
         getActiveSubclusters();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
index 0541df4..3d39d72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/FederationAMRMProxyPolicy.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -39,15 +40,16 @@ public interface FederationAMRMProxyPolicy
    *
    * @param resourceRequests the list of {@link ResourceRequest}s from the AM to
    *          be split
-   *
+   * @param timedOutSubClusters the set of sub-clusters that haven't had a
+   *          successful heart-beat response for a while.
    * @return map of sub-cluster as identified by {@link SubClusterId} to the
    *         list of {@link ResourceRequest}s that should be forwarded to it
-   *
    * @throws YarnException in case the request is malformed or no viable
    *           sub-clusters can be found.
    */
   Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
-      List<ResourceRequest> resourceRequests) throws YarnException;
+      List<ResourceRequest> resourceRequests,
+      Set<SubClusterId> timedOutSubClusters) throws YarnException;
 
   /**
    * This method should be invoked to notify the policy about responses being

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
index 57b6584..c4246cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -132,11 +131,8 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
   private SubClusterResolver resolver;
 
   private Map<SubClusterId, Resource> headroom;
-  private Map<SubClusterId, Long> lastHeartbeatTimeStamp;
-  private long subClusterTimeOut;
   private float hrAlpha;
   private FederationStateStoreFacade federationFacade;
-  private AllocationBookkeeper bookkeeper;
   private SubClusterId homeSubcluster;
 
   @Override
@@ -186,26 +182,12 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
 
     if (headroom == null) {
       headroom = new ConcurrentHashMap<>();
-      lastHeartbeatTimeStamp = new ConcurrentHashMap<>();
     }
     hrAlpha = policy.getHeadroomAlpha();
 
     this.federationFacade =
         policyContext.getFederationStateStoreFacade();
     this.homeSubcluster = policyContext.getHomeSubcluster();
-
-    this.subClusterTimeOut = this.federationFacade.getConf().getLong(
-        YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT,
-        YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT);
-    if (this.subClusterTimeOut <= 0) {
-      LOG.info(
-          "{} configured to be {}, should be positive. Using default of {}.",
-          YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT,
-          this.subClusterTimeOut,
-          YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT);
-      this.subClusterTimeOut =
-          YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT;
-    }
   }
 
   @Override
@@ -216,18 +198,18 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
       LOG.info("Subcluster {} updated with {} memory headroom", subClusterId,
           response.getAvailableResources().getMemorySize());
     }
-    lastHeartbeatTimeStamp.put(subClusterId, System.currentTimeMillis());
   }
 
   @Override
   public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
-      List<ResourceRequest> resourceRequests) throws YarnException {
+      List<ResourceRequest> resourceRequests,
+      Set<SubClusterId> timedOutSubClusters) throws YarnException {
 
     // object used to accumulate statistics about the answer, initialize with
     // active subclusters. Create a new instance per call because this method
     // can be called concurrently.
-    bookkeeper = new AllocationBookkeeper();
-    bookkeeper.reinitialize(federationFacade.getSubClusters(true));
+    AllocationBookkeeper bookkeeper = new AllocationBookkeeper();
+    bookkeeper.reinitialize(getActiveSubclusters(), timedOutSubClusters);
 
     List<ResourceRequest> nonLocalizedRequests =
         new ArrayList<ResourceRequest>();
@@ -298,15 +280,6 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
     // handle all non-localized requests (ANY)
     splitAnyRequests(nonLocalizedRequests, bookkeeper);
 
-    for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : bookkeeper
-        .getAnswer().entrySet()) {
-      // A new-cluster here will trigger new UAM luanch, which might take a long
-      // time. We don't want too many requests stuck in this UAM before it is
-      // ready and starts heartbeating
-      if (!lastHeartbeatTimeStamp.containsKey(entry.getKey())) {
-        lastHeartbeatTimeStamp.put(entry.getKey(), System.currentTimeMillis());
-      }
-    }
     return bookkeeper.getAnswer();
   }
 
@@ -540,8 +513,8 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
     private float totPolicyWeight = 0;
 
     private void reinitialize(
-        Map<SubClusterId, SubClusterInfo> activeSubclusters)
-        throws YarnException {
+        Map<SubClusterId, SubClusterInfo> activeSubclusters,
+        Set<SubClusterId> timedOutSubClusters) throws YarnException {
       if (activeSubclusters == null) {
         throw new YarnRuntimeException("null activeSubclusters received");
       }
@@ -573,17 +546,8 @@ public class LocalityMulticastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
       }
 
       Set<SubClusterId> tmpSCSet = new HashSet<>(activeAndEnabledSC);
-      for (Map.Entry<SubClusterId, Long> entry : lastHeartbeatTimeStamp
-          .entrySet()) {
-        long duration = System.currentTimeMillis() - entry.getValue();
-        if (duration > subClusterTimeOut) {
-          LOG.warn(
-              "Subcluster {} does not have a success heartbeat for {}s, "
-                  + "skip routing asks there for this request",
-              entry.getKey(), (double) duration / 1000);
-          tmpSCSet.remove(entry.getKey());
-        }
-      }
+      tmpSCSet.removeAll(timedOutSubClusters);
+
       if (tmpSCSet.size() < 1) {
         LOG.warn("All active and enabled subclusters have expired last "
             + "heartbeat time. Ignore the expiry check for this request");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
index 4500606..15c3424 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -48,7 +49,8 @@ public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
 
   @Override
   public Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
-      List<ResourceRequest> resourceRequests) throws YarnException {
+      List<ResourceRequest> resourceRequests,
+      Set<SubClusterId> timedOutSubClusters) throws YarnException {
     throw new FederationPolicyException("The policy configured for this queue "
         + "rejects all routing requests by construction.");
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
index 208a46c..6c20a16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/BaseFederationPoliciesTest.java
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.when;
 
 import java.nio.ByteBuffer;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -110,8 +111,8 @@ public abstract class BaseFederationPoliciesTest {
       String[] hosts = new String[] {"host1", "host2"};
       List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
           .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
-      ((FederationAMRMProxyPolicy) localPolicy)
-          .splitResourceRequests(resourceRequests);
+      ((FederationAMRMProxyPolicy) localPolicy).splitResourceRequests(
+          resourceRequests, new HashSet<SubClusterId>());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
index df5da85..52f36a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -28,7 +29,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.federation.policies.BaseFederationPoliciesTest;
 import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -71,8 +71,8 @@ public class TestBroadcastAMRMProxyFederationPolicy
         .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
     Assert.assertTrue(response.size() == 2);
     for (Map.Entry<SubClusterId, List<ResourceRequest>> entry : response
         .entrySet()) {
@@ -94,8 +94,8 @@ public class TestBroadcastAMRMProxyFederationPolicy
     List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
         .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
         SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
index c49ab60..10359e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
@@ -154,8 +154,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
     prepPolicyWithHeadroom(true);
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     // pretty print requests
     LOG.info("Initial headroom");
@@ -180,7 +180,7 @@ public class TestLocalityMulticastAMRMProxyPolicy
     ((FederationAMRMProxyPolicy) getPolicy())
         .notifyOfResponse(SubClusterId.newInstance("subcluster2"), ar);
     response = ((FederationAMRMProxyPolicy) getPolicy())
-        .splitResourceRequests(resourceRequests);
+        .splitResourceRequests(resourceRequests, new HashSet<SubClusterId>());
 
     LOG.info("After headroom update");
     prettyPrintRequests(response);
@@ -218,8 +218,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
     long tstart = System.currentTimeMillis();
     for (int i = 0; i < numIterations; i++) {
       Map<SubClusterId, List<ResourceRequest>> response =
-          ((FederationAMRMProxyPolicy) getPolicy())
-              .splitResourceRequests(resourceRequests);
+          ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+              resourceRequests, new HashSet<SubClusterId>());
       validateSplit(response, resourceRequests);
     }
     long tend = System.currentTimeMillis();
@@ -243,8 +243,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
     prepPolicyWithHeadroom(true);
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     // we expect all three to appear for a zero-sized ANY
 
@@ -279,8 +279,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
     prepPolicyWithHeadroom(true);
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     // pretty print requests
     prettyPrintRequests(response);
@@ -354,8 +354,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
     List<ResourceRequest> resourceRequests = createComplexRequest();
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     validateSplit(response, resourceRequests);
     prettyPrintRequests(response);
@@ -697,8 +697,8 @@ public class TestLocalityMulticastAMRMProxyPolicy
         ResourceRequest.ANY, 1024, 1, 1, 0, null, false));
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
 
     checkExpectedAllocation(response, "subcluster0", 3, 1);
     checkExpectedAllocation(response, "subcluster1", 1, 0);
@@ -717,7 +717,7 @@ public class TestLocalityMulticastAMRMProxyPolicy
         ResourceRequest.ANY, 1024, 1, 1, 100, null, false));
 
     response = ((FederationAMRMProxyPolicy) getPolicy())
-        .splitResourceRequests(resourceRequests);
+        .splitResourceRequests(resourceRequests, new HashSet<SubClusterId>());
 
     /*
      * Since node request is a cancel, it should not be considered associated
@@ -750,12 +750,13 @@ public class TestLocalityMulticastAMRMProxyPolicy
     initializePolicy(conf);
     List<ResourceRequest> resourceRequests = createSimpleRequest();
 
-    // Update the response timestamp for the first time
     prepPolicyWithHeadroom(true);
 
+    // For first time, no sub-cluster expired
+    Set<SubClusterId> expiredSCList = new HashSet<>();
     Map<SubClusterId, List<ResourceRequest>> response =
         ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+            .splitResourceRequests(resourceRequests, expiredSCList);
 
     // pretty print requests
     prettyPrintRequests(response);
@@ -776,11 +777,11 @@ public class TestLocalityMulticastAMRMProxyPolicy
 
     Thread.sleep(800);
 
-    // Update the response timestamp for the second time, skipping sc0 and sc5
-    prepPolicyWithHeadroom(false);
-
+    // For the second time, sc0 and sc5 expired
+    expiredSCList.add(SubClusterId.newInstance("subcluster0"));
+    expiredSCList.add(SubClusterId.newInstance("subcluster5"));
     response = ((FederationAMRMProxyPolicy) getPolicy())
-        .splitResourceRequests(resourceRequests);
+        .splitResourceRequests(resourceRequests, expiredSCList);
 
     // pretty print requests
     prettyPrintRequests(response);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
index 41e7fed..6555829 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestRejectAMRMProxyPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -69,8 +70,8 @@ public class TestRejectAMRMProxyPolicy
         .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
 
     Map<SubClusterId, List<ResourceRequest>> response =
-        ((FederationAMRMProxyPolicy) getPolicy())
-            .splitResourceRequests(resourceRequests);
+        ((FederationAMRMProxyPolicy) getPolicy()).splitResourceRequests(
+            resourceRequests, new HashSet<SubClusterId>());
   }
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index ae9f78d..dc10f95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.PreemptionContract;
 import org.apache.hadoop.yarn.api.records.PreemptionMessage;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
@@ -163,10 +165,22 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
 
   /**
    * Stores the AllocateResponses that are received asynchronously from all the
-   * sub-cluster resource managers, including home RM.
+   * sub-cluster resource managers, including home RM, but not merged and
+   * returned back to AM yet.
    */
   private Map<SubClusterId, List<AllocateResponse>> asyncResponseSink;
 
+  /**
+   * Remembers the last allocate response from all known sub-clusters. This is
+   * used together with sub-cluster timeout to assemble entries about
+   * cluster-wide info (e.g. AvailableResource, NumClusterNodes) in the allocate
+   * response back to AM.
+   */
+  private Map<SubClusterId, AllocateResponse> lastSCResponse;
+
+  /**
+   * The async UAM registration result that is not consumed yet.
+   */
   private Map<SubClusterId, RegisterApplicationMasterResponse> uamRegistrations;
 
   // For unit test synchronization
@@ -216,6 +230,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
 
   private SubClusterResolver subClusterResolver;
 
+  /**
+   * Records the last time a successful heartbeat response received from a known
+   * sub-cluster. lastHeartbeatTimeStamp.keySet() should be in sync with
+   * uamPool.getAllUAMIds().
+   */
+  private Map<SubClusterId, Long> lastSCResponseTime;
+  private long subClusterTimeOut;
+
+  private long lastAMHeartbeatTime;
+
   /** The policy used to split requests among sub-clusters. */
   private FederationAMRMProxyPolicy policyInterpreter;
 
@@ -232,6 +256,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   public FederationInterceptor() {
     this.containerIdToSubClusterIdMap = new ConcurrentHashMap<>();
     this.asyncResponseSink = new ConcurrentHashMap<>();
+    this.lastSCResponse = new ConcurrentHashMap<>();
     this.uamRegistrations = new ConcurrentHashMap<>();
     this.uamRegisterFutures = new ConcurrentHashMap<>();
     this.threadpool = Executors.newCachedThreadPool();
@@ -241,6 +266,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     this.amRegistrationResponse = null;
     this.justRecovered = false;
     this.finishAMCalled = false;
+    this.lastSCResponseTime = new ConcurrentHashMap<>();
+    this.lastAMHeartbeatTime = this.clock.getTime();
   }
 
   /**
@@ -310,6 +337,19 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     this.heartbeatMaxWaitTimeMs =
         conf.getLong(YarnConfiguration.FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS,
             YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_HB_MAX_WAIT_MS);
+
+    this.subClusterTimeOut =
+        conf.getLong(YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT,
+            YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT);
+    if (this.subClusterTimeOut <= 0) {
+      LOG.info(
+          "{} configured to be {}, should be positive. Using default of {}.",
+          YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT,
+          this.subClusterTimeOut,
+          YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT);
+      this.subClusterTimeOut =
+          YarnConfiguration.DEFAULT_FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT;
+    }
   }
 
   @Override
@@ -394,6 +434,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
               this.uamPool.registerApplicationMaster(subClusterId.getId(),
                   this.amRegistrationRequest);
 
+          // Set sub-cluster to be timed out initially
+          lastSCResponseTime.put(subClusterId,
+              clock.getTime() - subClusterTimeOut);
+
           // Running containers from secondary RMs
           for (Container container : response
               .getContainersFromPreviousAttempts()) {
@@ -580,6 +624,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       throws YarnException, IOException {
     Preconditions.checkArgument(this.policyInterpreter != null,
         "Allocate should be called after registerApplicationMaster");
+    this.lastAMHeartbeatTime = this.clock.getTime();
 
     if (this.justRecovered) {
       throw new ApplicationMasterNotRegisteredException(
@@ -644,8 +689,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       }
 
       // Prepare the response to AM
-      AllocateResponse response =
-          RECORD_FACTORY.newRecordInstance(AllocateResponse.class);
+      AllocateResponse response = generateBaseAllocationResponse();
 
       // Merge all responses from response sink
       mergeAllocateResponses(response);
@@ -970,6 +1014,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
                 response = uamPool.registerApplicationMaster(
                     subClusterId.getId(), amRegistrationRequest);
 
+                // Set sub-cluster to be timed out initially
+                lastSCResponseTime.put(subClusterId,
+                    clock.getTime() - subClusterTimeOut);
+
                 if (response != null
                     && response.getContainersFromPreviousAttempts() != null) {
                   cacheAllocatedContainers(
@@ -1172,6 +1220,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       if (!subClusterId.equals(this.homeSubClusterId)
           && !this.uamPool.hasUAMId(subClusterId.getId())) {
         newSubClusters.add(subClusterId);
+
+        // Set sub-cluster to be timed out initially
+        lastSCResponseTime.put(subClusterId,
+            clock.getTime() - subClusterTimeOut);
       }
     }
 
@@ -1245,6 +1297,38 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
   }
 
   /**
+   * Prepare the base allocation response. Use lastSCResponse and
+   * lastHeartbeatTimeStamp to assemble entries about cluster-wide info, e.g.
+   * AvailableResource, NumClusterNodes.
+   */
+  protected AllocateResponse generateBaseAllocationResponse() {
+    AllocateResponse baseResponse =
+        RECORD_FACTORY.newRecordInstance(AllocateResponse.class);
+
+    baseResponse.setAvailableResources(Resource.newInstance(0, 0));
+    baseResponse.setNumClusterNodes(0);
+
+    Set<SubClusterId> expiredSC = getTimedOutSCs(false);
+    for (Entry<SubClusterId, AllocateResponse> entry : lastSCResponse
+        .entrySet()) {
+      if (expiredSC.contains(entry.getKey())) {
+        // Skip expired sub-clusters
+        continue;
+      }
+      AllocateResponse response = entry.getValue();
+
+      if (response.getAvailableResources() != null) {
+        baseResponse.setAvailableResources(
+            Resources.add(baseResponse.getAvailableResources(),
+                response.getAvailableResources()));
+      }
+      baseResponse.setNumClusterNodes(
+          baseResponse.getNumClusterNodes() + response.getNumClusterNodes());
+    }
+    return baseResponse;
+  }
+
+  /**
    * Merge the responses from all sub-clusters that we received asynchronously
    * and keeps track of the containers received from each sub-cluster resource
    * managers.
@@ -1345,17 +1429,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
       }
     }
 
-    if (otherResponse.getAvailableResources() != null) {
-      if (homeResponse.getAvailableResources() != null) {
-        homeResponse.setAvailableResources(
-            Resources.add(homeResponse.getAvailableResources(),
-                otherResponse.getAvailableResources()));
-      } else {
-        homeResponse
-            .setAvailableResources(otherResponse.getAvailableResources());
-      }
-    }
-
     if (!isNullOrEmpty(otherResponse.getCompletedContainersStatuses())) {
       if (!isNullOrEmpty(homeResponse.getCompletedContainersStatuses())) {
         homeResponse.getCompletedContainersStatuses()
@@ -1520,6 +1593,29 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     return request;
   }
 
+  protected Set<SubClusterId> getTimedOutSCs(boolean verbose) {
+    Set<SubClusterId> timedOutSCs = new HashSet<>();
+    for (Map.Entry<SubClusterId, Long> entry : this.lastSCResponseTime
+        .entrySet()) {
+      if (entry.getValue() > this.lastAMHeartbeatTime) {
+        // AM haven't heartbeat to us (and thus we to all SCs) for a long time,
+        // should not consider the SC as timed out
+        continue;
+      }
+      long duration = this.clock.getTime() - entry.getValue();
+      if (duration > this.subClusterTimeOut) {
+        if (verbose) {
+          LOG.warn(
+              "Subcluster {} doesn't have a successful heartbeat"
+                  + " for {} seconds for {}",
+              entry.getKey(), (double) duration / 1000, this.attemptId);
+        }
+        timedOutSCs.add(entry.getKey());
+      }
+    }
+    return timedOutSCs;
+  }
+
   /**
    * Check to see if the specified containerId exists in the cache and log an
    * error if not found.
@@ -1553,7 +1649,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    */
   protected Map<SubClusterId, List<ResourceRequest>> splitResourceRequests(
       List<ResourceRequest> askList) throws YarnException {
-    return this.policyInterpreter.splitResourceRequests(askList);
+    return policyInterpreter.splitResourceRequests(askList,
+        getTimedOutSCs(true));
   }
 
   @VisibleForTesting
@@ -1602,6 +1699,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
         // Notify main thread about the response arrival
         asyncResponseSink.notifyAll();
       }
+      lastSCResponse.put(subClusterId, response);
+      lastSCResponseTime.put(subClusterId, clock.getTime());
 
       // Notify policy of allocate response
       try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b4455d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index ec75cfd..48b7bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -160,6 +160,10 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
     // Disable StateStoreFacade cache
     conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
 
+    // Set sub-cluster timeout to 500ms
+    conf.setLong(YarnConfiguration.FEDERATION_AMRMPROXY_SUBCLUSTER_TIMEOUT,
+        500);
+
     return conf;
   }
 
@@ -568,6 +572,8 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
         interceptor.recover(recoveredDataMap);
 
         Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+        // SC1 should be initialized to be timed out
+        Assert.assertEquals(1, interceptor.getTimedOutSCs(true).size());
 
         // The first allocate call expects a fail-over exception and re-register
         try {
@@ -741,6 +747,60 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
   }
 
   @Test
+  public void testSubClusterTimeOut() throws Exception {
+    UserGroupInformation ugi =
+        interceptor.getUGIWithToken(interceptor.getAttemptId());
+    ugi.doAs(new PrivilegedExceptionAction<Object>() {
+      @Override
+      public Object run() throws Exception {
+        // Register the application first time
+        RegisterApplicationMasterRequest registerReq =
+            Records.newRecord(RegisterApplicationMasterRequest.class);
+        registerReq.setHost(Integer.toString(testAppId));
+        registerReq.setRpcPort(0);
+        registerReq.setTrackingUrl("");
+        RegisterApplicationMasterResponse registerResponse =
+            interceptor.registerApplicationMaster(registerReq);
+        Assert.assertNotNull(registerResponse);
+        lastResponseId = 0;
+
+        registerSubCluster(SubClusterId.newInstance("SC-1"));
+
+        getContainersAndAssert(1, 1);
+
+        AllocateResponse allocateResponse =
+            interceptor.generateBaseAllocationResponse();
+        Assert.assertEquals(2, allocateResponse.getNumClusterNodes());
+        Assert.assertEquals(0, interceptor.getTimedOutSCs(true).size());
+
+        // Let all SC timeout (home and SC-1), without an allocate from AM
+        Thread.sleep(800);
+
+        // Should not be considered timeout, because there's no recent AM
+        // heartbeat
+        allocateResponse = interceptor.generateBaseAllocationResponse();
+        Assert.assertEquals(2, allocateResponse.getNumClusterNodes());
+        Assert.assertEquals(0, interceptor.getTimedOutSCs(true).size());
+
+        // Generate a duplicate heartbeat from AM, so that it won't really
+        // trigger an heartbeat to all SC
+        AllocateRequest allocateRequest =
+            Records.newRecord(AllocateRequest.class);
+        // Set to lastResponseId - 1 so that it will be considered a duplicate
+        // heartbeat and thus not forwarded to all SCs
+        allocateRequest.setResponseId(lastResponseId - 1);
+        interceptor.allocate(allocateRequest);
+
+        // Should be considered timeout
+        allocateResponse = interceptor.generateBaseAllocationResponse();
+        Assert.assertEquals(0, allocateResponse.getNumClusterNodes());
+        Assert.assertEquals(2, interceptor.getTimedOutSCs(true).size());
+        return null;
+      }
+    });
+  }
+
+  @Test
   public void testSecondAttempt() throws Exception {
     final RegisterApplicationMasterRequest registerReq =
         Records.newRecord(RegisterApplicationMasterRequest.class);
@@ -803,6 +863,8 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
         int numberOfContainers = 3;
         // Should re-attach secondaries and get the three running containers
         Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+        // SC1 should be initialized to be timed out
+        Assert.assertEquals(1, interceptor.getTimedOutSCs(true).size());
         Assert.assertEquals(numberOfContainers,
             registerResponse.getContainersFromPreviousAttempts().size());
 
@@ -831,5 +893,4 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
       }
     });
   }
-
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[07/50] [abbrv] hadoop git commit: YARN-6989 Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed by Abhishek Modi

Posted by jh...@apache.org.
YARN-6989 Ensure timeline service v2 codebase gets UGI from HttpServletRequest in a consistent way. Contributed by Abhishek Modi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6cc6858
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6cc6858
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6cc6858

Branch: refs/heads/branch-2-jhung-test
Commit: c6cc685881ee4e3b445116ea5a11af02ed360190
Parents: c102ebf
Author: Vrushali C <vr...@apache.org>
Authored: Wed Oct 10 15:25:58 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../reader/TimelineReaderWebServicesUtils.java  | 21 ++------------------
 ...elineReaderWhitelistAuthorizationFilter.java |  2 +-
 2 files changed, 3 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cc6858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index ae19b21..02f510a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -269,25 +269,8 @@ public final class TimelineReaderWebServicesUtils {
    * @return UGI.
    */
   public static UserGroupInformation getUser(HttpServletRequest req) {
-    return getCallerUserGroupInformation(req, false);
-  }
-
-  /**
-   * Get UGI from the HTTP request.
-   *
-   * @param hsr HTTP request.
-   * @param usePrincipal if true, use principal name else use remote user name
-   * @return UGI.
-   */
-  public static UserGroupInformation getCallerUserGroupInformation(
-      HttpServletRequest hsr, boolean usePrincipal) {
-
-    String remoteUser = hsr.getRemoteUser();
-    if (usePrincipal) {
-      Principal princ = hsr.getUserPrincipal();
-      remoteUser = princ == null ? null : princ.getName();
-    }
-
+    Principal princ = req.getUserPrincipal();
+    String remoteUser = princ == null ? null : princ.getName();
     UserGroupInformation callerUGI = null;
     if (remoteUser != null) {
       callerUGI = UserGroupInformation.createRemoteUser(remoteUser);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cc6858/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
index d3f63e5..4ed3f23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderWhitelistAuthorizationFilter.java
@@ -67,7 +67,7 @@ public class TimelineReaderWhitelistAuthorizationFilter implements Filter {
 
     if (isWhitelistReadAuthEnabled) {
       UserGroupInformation callerUGI = TimelineReaderWebServicesUtils
-          .getCallerUserGroupInformation(httpRequest, true);
+          .getUser(httpRequest);
       if (callerUGI == null) {
         String msg = "Unable to obtain user name, user not authenticated";
         throw new AuthorizationException(msg);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[09/50] [abbrv] hadoop git commit: HADOOP-15859. ZStandardDecompressor.c mistakes a class for an instance. Contributed by Jason Lowe

Posted by jh...@apache.org.
HADOOP-15859. ZStandardDecompressor.c mistakes a class for an instance. Contributed by Jason Lowe

(cherry picked from commit 9abda83947a5babfe5a650b3409ad952f6782105)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03ff0f17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03ff0f17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03ff0f17

Branch: refs/heads/branch-2-jhung-test
Commit: 03ff0f171eba341f54f53f6ae7283b0657504f04
Parents: 04ca639
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Oct 17 14:38:42 2018 -0500
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/io/compress/zstd/ZStandardCompressor.java     |  2 +-
 .../apache/hadoop/io/compress/zstd/ZStandardCompressor.c | 10 +++++-----
 .../hadoop/io/compress/zstd/ZStandardDecompressor.c      | 11 +++++------
 3 files changed, 11 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/03ff0f17/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
index eb2121a..7445502 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.java
@@ -298,7 +298,7 @@ public class ZStandardCompressor implements Compressor {
   private native static void init(int level, long stream);
   private native int deflateBytesDirect(ByteBuffer src, int srcOffset,
       int srcLen, ByteBuffer dst, int dstLen);
-  private static native int getStreamSize();
+  private native static int getStreamSize();
   private native static void end(long strm);
   private native static void initIDs();
   public native static String getLibraryName();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/03ff0f17/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
index 04f2a3e..055683a 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardCompressor.c
@@ -139,7 +139,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompress
 }
 
 // Create the compression stream
-JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_create (JNIEnv *env, jobject this) {
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_create (JNIEnv *env, jclass clazz) {
     ZSTD_CStream* const stream =  dlsym_ZSTD_createCStream();
     if (stream == NULL) {
         THROW(env, "java/lang/InternalError", "Error creating the stream");
@@ -149,7 +149,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompres
 }
 
 // Initialize the compression stream
-JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_init (JNIEnv *env, jobject this, jint level, jlong stream) {
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_init (JNIEnv *env, jclass clazz, jint level, jlong stream) {
     size_t result = dlsym_ZSTD_initCStream((ZSTD_CStream *) stream, level);
     if (dlsym_ZSTD_isError(result)) {
         THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
@@ -158,7 +158,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompress
 }
 
 // free the compression stream
-JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_end (JNIEnv *env, jobject this, jlong stream) {
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_end (JNIEnv *env, jclass clazz, jlong stream) {
     size_t result = dlsym_ZSTD_freeCStream((ZSTD_CStream *) stream);
     if (dlsym_ZSTD_isError(result)) {
         THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
@@ -227,7 +227,7 @@ JNIEXPORT jint Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_defla
 }
 
 JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_getLibraryName
-(JNIEnv *env, jclass class) {
+(JNIEnv *env, jclass clazz) {
 #ifdef UNIX
     if (dlsym_ZSTD_isError) {
         Dl_info dl_info;
@@ -250,7 +250,7 @@ JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompr
 
 // returns the max size of the recommended input and output buffers
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardCompressor_getStreamSize
-(JNIEnv *env, jobject this) {
+(JNIEnv *env, jclass clazz) {
     int x = (int) dlsym_ZSTD_CStreamInSize();
     int y = (int) dlsym_ZSTD_CStreamOutSize();
     return (x >= y) ? x : y;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/03ff0f17/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
index e75a6ef..b97dd17 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zstd/ZStandardDecompressor.c
@@ -130,7 +130,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
     ZStandardDecompressor_remaining = (*env)->GetFieldID(env, clazz, "remaining", "I");
 }
 
-JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_create(JNIEnv *env, jobject this) {
+JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_create(JNIEnv *env, jclass clazz) {
     ZSTD_DStream * stream = dlsym_ZSTD_createDStream();
     if (stream == NULL) {
         THROW(env, "java/lang/InternalError", "Error creating stream");
@@ -139,17 +139,16 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompr
     return (jlong) stream;
 }
 
-JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_init(JNIEnv *env, jobject this, jlong stream) {
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_init(JNIEnv *env, jclass clazz, jlong stream) {
     size_t result = dlsym_ZSTD_initDStream((ZSTD_DStream *) stream);
     if (dlsym_ZSTD_isError(result)) {
         THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
         return;
     }
-    (*env)->SetIntField(env, this, ZStandardDecompressor_remaining, 0);
 }
 
 
-JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_free(JNIEnv *env, jclass obj, jlong stream) {
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_free(JNIEnv *env, jclass clazz, jlong stream) {
     size_t result = dlsym_ZSTD_freeDStream((ZSTD_DStream *) stream);
     if (dlsym_ZSTD_isError(result)) {
         THROW(env, "java/lang/InternalError", dlsym_ZSTD_getErrorName(result));
@@ -209,10 +208,10 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompre
 
 // returns the max size of the recommended input and output buffers
 JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zstd_ZStandardDecompressor_getStreamSize
-(JNIEnv *env, jclass obj) {
+(JNIEnv *env, jclass clazz) {
     int x = (int) dlsym_ZSTD_DStreamInSize();
     int y = (int) dlsym_ZSTD_DStreamOutSize();
     return (x >= y) ? x : y;
 }
 
-#endif //define HADOOP_ZSTD_LIBRARY
\ No newline at end of file
+#endif //define HADOOP_ZSTD_LIBRARY


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[37/50] [abbrv] hadoop git commit: HDFS-14056. Fix error messages in HDFS-12716. Contributed by Ayush Saxena.

Posted by jh...@apache.org.
HDFS-14056. Fix error messages in HDFS-12716. Contributed by Ayush Saxena.

(cherry picked from commit 886b935630bf2d46f4c6773de013c093d2e44c38)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/762df75d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/762df75d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/762df75d

Branch: refs/heads/branch-2-jhung-test
Commit: 762df75dd702711513717dfa5a65bb2da31bc3af
Parents: 0cebbfd
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Nov 16 18:01:13 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hdfs/server/datanode/DataNode.java   | 5 +++--
 .../hadoop-hdfs/src/main/resources/hdfs-default.xml             | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/762df75d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 80a7ca2..1fab84a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -294,7 +294,8 @@ public class DataNode extends ReconfigurableBase
 
   static final int CURRENT_BLOCK_FORMAT_VERSION = 1;
   public static final int MAX_VOLUME_FAILURE_TOLERATED_LIMIT = -1;
-  public static final String MAX_VOLUME_FAILURES_TOLERATED_MSG = "should be greater than -1";
+  public static final String MAX_VOLUME_FAILURES_TOLERATED_MSG =
+      "should be greater than or equal to -1";
 
 
   /** A list of property that are reconfigurable at runtime. */
@@ -1362,7 +1363,7 @@ public class DataNode extends ReconfigurableBase
         || volFailuresTolerated >= volsConfigured) {
       throw new DiskErrorException("Invalid value configured for "
           + "dfs.datanode.failed.volumes.tolerated - " + volFailuresTolerated
-          + ". Value configured is either greater than -1 or >= "
+          + ". Value configured is either less than -1 or >= "
           + "to the number of configured volumes (" + volsConfigured + ").");
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/762df75d/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 5802fce..0ce3bd9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1222,8 +1222,8 @@
   <description>The number of volumes that are allowed to
   fail before a datanode stops offering service. By default
   any volume failure will cause a datanode to shutdown.
-  The range of the value is -1 now, -1 represents the minimum
-  of volume valids is 1.
+  The value should be greater than or equal to -1 , -1 represents minimum
+  1 valid volume.
   </description>
 </property>
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[21/50] [abbrv] hadoop git commit: HADOOP-15866. Renamed HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT keys break compatibility. Contributed by Wei-Chiu Chuang.

Posted by jh...@apache.org.
HADOOP-15866. Renamed HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT keys break compatibility.
Contributed by Wei-Chiu Chuang.

(cherry picked from commit 52cb766ad0bbea378a529fcff3cf372cf0a33cde)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d021000
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d021000
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d021000

Branch: refs/heads/branch-2-jhung-test
Commit: 6d021000f901f18dae73a794c0e57c8bf759eb9f
Parents: 091c7cf
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 22 11:06:56 2018 +0100
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/fs/CommonConfigurationKeysPublic.java | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d021000/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index df4dfbe..9adb9ba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -557,13 +557,28 @@ public class CommonConfigurationKeysPublic {
   public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY =
       "hadoop.security.groups.shell.command.timeout";
   /**
+   * @deprecated use
+   * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY}
+   * instead.
+   */
+  public static final String HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS =
+      HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_KEY;
+  /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
    * core-default.xml</a>
    */
   public static final long
-          HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT =
-          0L;
+      HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT =
+      0L;
+  /**
+   * @deprecated use
+   * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT}
+   * instead.
+   */
+  public static final long
+      HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_SECS_DEFAULT =
+      HADOOP_SECURITY_GROUP_SHELL_COMMAND_TIMEOUT_DEFAULT;
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[14/50] [abbrv] hadoop git commit: Fix potential FSImage corruption. Contributed by Daryn Sharp.

Posted by jh...@apache.org.
Fix potential FSImage corruption. Contributed by Daryn Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04ca639a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04ca639a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04ca639a

Branch: refs/heads/branch-2-jhung-test
Commit: 04ca639af55c11b271864d70157ad2b58e395fc1
Parents: 6e5d052
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Oct 15 16:04:34 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../apache/hadoop/hdfs/util/LongBitFormat.java  |   8 +
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   6 +
 .../server/namenode/AclEntryStatusFormat.java   | 109 +++++-----
 .../hdfs/server/namenode/FSDirectory.java       |   2 +
 .../server/namenode/FSImageFormatPBINode.java   |  99 +++------
 .../server/namenode/FSImageFormatProtobuf.java  |  26 ++-
 .../namenode/INodeWithAdditionalFields.java     |  36 +++-
 .../server/namenode/SerialNumberManager.java    | 200 +++++++++++++++++--
 .../hdfs/server/namenode/SerialNumberMap.java   |  43 +++-
 .../hdfs/server/namenode/XAttrFormat.java       |  95 +++++----
 .../hdfs/server/namenode/XAttrStorage.java      |  11 -
 .../tools/offlineImageViewer/FSImageLoader.java |  18 +-
 .../offlineImageViewer/PBImageTextWriter.java   |   3 +-
 .../offlineImageViewer/PBImageXmlWriter.java    |  22 +-
 .../hadoop-hdfs/src/main/proto/fsimage.proto    |   1 +
 .../TestCommitBlockSynchronization.java         |   6 +-
 16 files changed, 436 insertions(+), 249 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
index 51ad08f..195fd8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java
@@ -26,6 +26,10 @@ import java.io.Serializable;
 public class LongBitFormat implements Serializable {
   private static final long serialVersionUID = 1L;
 
+  public interface Enum {
+    int getLength();
+  }
+
   private final String NAME;
   /** Bit offset */
   private final int OFFSET;
@@ -69,4 +73,8 @@ public class LongBitFormat implements Serializable {
   public long getMin() {
     return MIN;
   }
+
+  public int getLength() {
+    return LENGTH;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index edd99f2..81614e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -685,6 +685,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final long DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_DEFAULT =
       0;  //no throttling
 
+  // String table in the fsimage utilizes an expanded bit range.
+  public static final String DFS_IMAGE_EXPANDED_STRING_TABLES_KEY =
+      "dfs.image.string-tables.expanded";
+  public static final boolean DFS_IMAGE_EXPANDED_STRING_TABLES_DEFAULT =
+      false;
+
   // Image transfer timeout
   public static final String DFS_IMAGE_TRANSFER_TIMEOUT_KEY = "dfs.image.transfer.timeout";
   public static final int DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT = 60 * 1000;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
index 2c5b23b..e9e3e59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java
@@ -31,25 +31,23 @@ import com.google.common.collect.ImmutableList;
 /**
  * Class to pack an AclEntry into an integer. <br>
  * An ACL entry is represented by a 32-bit integer in Big Endian format. <br>
- * The bits can be divided in four segments: <br>
- * [0:1) || [1:3) || [3:6) || [6:7) || [7:32) <br>
- * <br>
- * [0:1) -- the scope of the entry (AclEntryScope) <br>
- * [1:3) -- the type of the entry (AclEntryType) <br>
- * [3:6) -- the permission of the entry (FsAction) <br>
- * [6:7) -- A flag to indicate whether Named entry or not <br>
- * [7:8) -- Reserved <br>
- * [8:32) -- the name of the entry, which is an ID that points to a <br>
- * string in the StringTableSection. <br>
+ *
+ * Note:  this format is used both in-memory and on-disk.  Changes will be
+ * incompatible.
+ *
  */
-public enum AclEntryStatusFormat {
+public enum AclEntryStatusFormat implements LongBitFormat.Enum {
+
+  PERMISSION(null, 3),
+  TYPE(PERMISSION.BITS, 2),
+  SCOPE(TYPE.BITS, 1),
+  NAME(SCOPE.BITS, 24);
 
-  SCOPE(null, 1),
-  TYPE(SCOPE.BITS, 2),
-  PERMISSION(TYPE.BITS, 3),
-  NAMED_ENTRY_CHECK(PERMISSION.BITS, 1),
-  RESERVED(NAMED_ENTRY_CHECK.BITS, 1),
-  NAME(RESERVED.BITS, 24);
+  private static final FsAction[] FSACTION_VALUES = FsAction.values();
+  private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES =
+      AclEntryScope.values();
+  private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES =
+      AclEntryType.values();
 
   private final LongBitFormat BITS;
 
@@ -59,30 +57,29 @@ public enum AclEntryStatusFormat {
 
   static AclEntryScope getScope(int aclEntry) {
     int ordinal = (int) SCOPE.BITS.retrieve(aclEntry);
-    return AclEntryScope.values()[ordinal];
+    return ACL_ENTRY_SCOPE_VALUES[ordinal];
   }
 
   static AclEntryType getType(int aclEntry) {
     int ordinal = (int) TYPE.BITS.retrieve(aclEntry);
-    return AclEntryType.values()[ordinal];
+    return ACL_ENTRY_TYPE_VALUES[ordinal];
   }
 
   static FsAction getPermission(int aclEntry) {
     int ordinal = (int) PERMISSION.BITS.retrieve(aclEntry);
-    return FsAction.values()[ordinal];
+    return FSACTION_VALUES[ordinal];
   }
 
   static String getName(int aclEntry) {
-    int nameExists = (int) NAMED_ENTRY_CHECK.BITS.retrieve(aclEntry);
-    if (nameExists == 0) {
-      return null;
-    }
-    int id = (int) NAME.BITS.retrieve(aclEntry);
-    AclEntryType type = getType(aclEntry);
-    if (type == AclEntryType.USER) {
-      return SerialNumberManager.INSTANCE.getUser(id);
-    } else if (type == AclEntryType.GROUP) {
-      return SerialNumberManager.INSTANCE.getGroup(id);
+    return getName(aclEntry, null);
+  }
+
+  static String getName(int aclEntry,
+                        SerialNumberManager.StringTable stringTable) {
+    SerialNumberManager snm = getSerialNumberManager(getType(aclEntry));
+    if (snm != null) {
+      int nid = (int)NAME.BITS.retrieve(aclEntry);
+      return snm.getString(nid, stringTable);
     }
     return null;
   }
@@ -94,29 +91,26 @@ public enum AclEntryStatusFormat {
     aclEntryInt = TYPE.BITS.combine(aclEntry.getType().ordinal(), aclEntryInt);
     aclEntryInt = PERMISSION.BITS.combine(aclEntry.getPermission().ordinal(),
         aclEntryInt);
-    if (aclEntry.getName() != null) {
-      aclEntryInt = NAMED_ENTRY_CHECK.BITS.combine(1, aclEntryInt);
-      if (aclEntry.getType() == AclEntryType.USER) {
-        int userId = SerialNumberManager.INSTANCE.getUserSerialNumber(aclEntry
-            .getName());
-        aclEntryInt = NAME.BITS.combine(userId, aclEntryInt);
-      } else if (aclEntry.getType() == AclEntryType.GROUP) {
-        int groupId = SerialNumberManager.INSTANCE
-            .getGroupSerialNumber(aclEntry.getName());
-        aclEntryInt = NAME.BITS.combine(groupId, aclEntryInt);
-      }
+    SerialNumberManager snm = getSerialNumberManager(aclEntry.getType());
+    if (snm != null) {
+      int nid = snm.getSerialNumber(aclEntry.getName());
+      aclEntryInt = NAME.BITS.combine(nid, aclEntryInt);
     }
     return (int) aclEntryInt;
   }
 
   static AclEntry toAclEntry(int aclEntry) {
-    AclEntry.Builder builder = new AclEntry.Builder();
-    builder.setScope(getScope(aclEntry)).setType(getType(aclEntry))
-        .setPermission(getPermission(aclEntry));
-    if (getName(aclEntry) != null) {
-      builder.setName(getName(aclEntry));
-    }
-    return builder.build();
+    return toAclEntry(aclEntry, null);
+  }
+
+  static AclEntry toAclEntry(int aclEntry,
+                             SerialNumberManager.StringTable stringTable) {
+    return new AclEntry.Builder()
+        .setScope(getScope(aclEntry))
+        .setType(getType(aclEntry))
+        .setPermission(getPermission(aclEntry))
+        .setName(getName(aclEntry, stringTable))
+        .build();
   }
 
   public static int[] toInt(List<AclEntry> aclEntries) {
@@ -127,12 +121,19 @@ public enum AclEntryStatusFormat {
     return entries;
   }
 
-  public static ImmutableList<AclEntry> toAclEntries(int[] entries) {
-    ImmutableList.Builder<AclEntry> b = new ImmutableList.Builder<AclEntry>();
-    for (int entry : entries) {
-      AclEntry aclEntry = toAclEntry(entry);
-      b.add(aclEntry);
+  private static SerialNumberManager getSerialNumberManager(AclEntryType type) {
+    switch (type) {
+      case USER:
+        return SerialNumberManager.USER;
+      case GROUP:
+        return SerialNumberManager.GROUP;
+      default:
+        return null;
     }
-    return b.build();
+  }
+
+  @Override
+  public int getLength() {
+    return BITS.getLength();
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 939445a..cf9f1b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -255,6 +255,8 @@ public class FSDirectory implements Closeable {
   };
 
   FSDirectory(FSNamesystem ns, Configuration conf) throws IOException {
+    // used to enable/disable the use of expanded string tables.
+    SerialNumberManager.initialize(conf);
     this.dirLock = new ReentrantReadWriteLock(true); // fair
     this.inodeId = new INodeId();
     rootDir = createRoot(ns);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index ed87d05..4bed197 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -31,10 +31,6 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.permission.AclEntry;
-import org.apache.hadoop.fs.permission.AclEntryScope;
-import org.apache.hadoop.fs.permission.AclEntryType;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.fs.XAttr;
@@ -57,6 +53,8 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCom
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto;
+import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields.PermissionStatusFormat;
+import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager.StringTable;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase;
 import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
@@ -65,15 +63,11 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
 import org.apache.hadoop.hdfs.util.EnumCounters;
 import org.apache.hadoop.hdfs.util.ReadOnlyList;
 
-import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.ByteString;
 
 @InterfaceAudience.Private
 public final class FSImageFormatPBINode {
-  private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1;
-  private final static int USER_STRID_OFFSET = 40;
-  private final static int GROUP_STRID_OFFSET = 16;
   private static final Log LOG = LogFactory.getLog(FSImageFormatPBINode.class);
 
   public static final int ACL_ENTRY_NAME_MASK = (1 << 24) - 1;
@@ -81,13 +75,6 @@ public final class FSImageFormatPBINode {
   public static final int ACL_ENTRY_TYPE_OFFSET = 3;
   public static final int ACL_ENTRY_SCOPE_OFFSET = 5;
   public static final int ACL_ENTRY_PERM_MASK = 7;
-  private static final int ACL_ENTRY_TYPE_MASK = 3;
-  private static final int ACL_ENTRY_SCOPE_MASK = 1;
-  private static final FsAction[] FSACTION_VALUES = FsAction.values();
-  private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = AclEntryScope
-      .values();
-  private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType
-      .values();
   
   public static final int XATTR_NAMESPACE_MASK = 3;
   public static final int XATTR_NAMESPACE_OFFSET = 30;
@@ -98,53 +85,33 @@ public final class FSImageFormatPBINode {
   public static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
   public static final int XATTR_NAMESPACE_EXT_MASK = 1;
 
-  private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
-      XAttr.NameSpace.values();
-  
-
+  // the loader must decode all fields referencing serial number based fields
+  // via to<Item> methods with the string table.
   public final static class Loader {
     public static PermissionStatus loadPermission(long id,
-        final String[] stringTable) {
-      short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1));
-      int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK);
-      int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK);
-      return new PermissionStatus(stringTable[usid], stringTable[gsid],
-          new FsPermission(perm));
+        final StringTable stringTable) {
+      return PermissionStatusFormat.toPermissionStatus(id, stringTable);
     }
 
     public static ImmutableList<AclEntry> loadAclEntries(
-        AclFeatureProto proto, final String[] stringTable) {
+        AclFeatureProto proto, final StringTable stringTable) {
       ImmutableList.Builder<AclEntry> b = ImmutableList.builder();
       for (int v : proto.getEntriesList()) {
-        int p = v & ACL_ENTRY_PERM_MASK;
-        int t = (v >> ACL_ENTRY_TYPE_OFFSET) & ACL_ENTRY_TYPE_MASK;
-        int s = (v >> ACL_ENTRY_SCOPE_OFFSET) & ACL_ENTRY_SCOPE_MASK;
-        int nid = (v >> ACL_ENTRY_NAME_OFFSET) & ACL_ENTRY_NAME_MASK;
-        String name = stringTable[nid];
-        b.add(new AclEntry.Builder().setName(name)
-            .setPermission(FSACTION_VALUES[p])
-            .setScope(ACL_ENTRY_SCOPE_VALUES[s])
-            .setType(ACL_ENTRY_TYPE_VALUES[t]).build());
+        b.add(AclEntryStatusFormat.toAclEntry(v, stringTable));
       }
       return b.build();
     }
     
     public static List<XAttr> loadXAttrs(
-        XAttrFeatureProto proto, final String[] stringTable) {
+        XAttrFeatureProto proto, final StringTable stringTable) {
       List<XAttr> b = new ArrayList<>();
       for (XAttrCompactProto xAttrCompactProto : proto.getXAttrsList()) {
         int v = xAttrCompactProto.getName();
-        int nid = (v >> XATTR_NAME_OFFSET) & XATTR_NAME_MASK;
-        int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK;
-        ns |=
-            ((v >> XATTR_NAMESPACE_EXT_OFFSET) & XATTR_NAMESPACE_EXT_MASK) << 2;
-        String name = stringTable[nid];
         byte[] value = null;
         if (xAttrCompactProto.getValue() != null) {
           value = xAttrCompactProto.getValue().toByteArray();
         }
-        b.add(new XAttr.Builder().setNameSpace(XATTR_NAMESPACE_VALUES[ns])
-            .setName(name).setValue(value).build());
+        b.add(XAttrFormat.toXAttr(v, value, stringTable));
       }
       
       return b;
@@ -406,46 +373,30 @@ public final class FSImageFormatPBINode {
     }
   }
 
+  // the saver can directly write out fields referencing serial numbers.
+  // the serial number maps will be compacted when loading.
   public final static class Saver {
     private long numImageErrors;
 
-    private static long buildPermissionStatus(INodeAttributes n,
-        final SaverContext.DeduplicationMap<String> stringMap) {
-      long userId = stringMap.getId(n.getUserName());
-      long groupId = stringMap.getId(n.getGroupName());
-      return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET)
-          | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET)
-          | n.getFsPermissionShort();
+    private static long buildPermissionStatus(INodeAttributes n) {
+      return n.getPermissionLong();
     }
 
-    private static AclFeatureProto.Builder buildAclEntries(AclFeature f,
-        final SaverContext.DeduplicationMap<String> map) {
+    private static AclFeatureProto.Builder buildAclEntries(AclFeature f) {
       AclFeatureProto.Builder b = AclFeatureProto.newBuilder();
       for (int pos = 0, e; pos < f.getEntriesSize(); pos++) {
         e = f.getEntryAt(pos);
-        int nameId = map.getId(AclEntryStatusFormat.getName(e));
-        int v = ((nameId & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET)
-            | (AclEntryStatusFormat.getType(e).ordinal() << ACL_ENTRY_TYPE_OFFSET)
-            | (AclEntryStatusFormat.getScope(e).ordinal() << ACL_ENTRY_SCOPE_OFFSET)
-            | (AclEntryStatusFormat.getPermission(e).ordinal());
-        b.addEntries(v);
+        b.addEntries(e);
       }
       return b;
     }
     
-    private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f,
-        final SaverContext.DeduplicationMap<String> stringMap) {
+    private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f) {
       XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder();
       for (XAttr a : f.getXAttrs()) {
         XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto.
             newBuilder();
-        int nsOrd = a.getNameSpace().ordinal();
-        Preconditions.checkArgument(nsOrd < 8, "Too many namespaces.");
-        int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET)
-            | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) <<
-                XATTR_NAME_OFFSET);
-        v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) <<
-            XATTR_NAMESPACE_EXT_OFFSET);
+        int v = XAttrFormat.toInt(a);
         xAttrCompactBuilder.setName(v);
         if (a.getValue() != null) {
           xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue()));
@@ -477,18 +428,18 @@ public final class FSImageFormatPBINode {
       INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder()
           .setAccessTime(file.getAccessTime())
           .setModificationTime(file.getModificationTime())
-          .setPermission(buildPermissionStatus(file, state.getStringMap()))
+          .setPermission(buildPermissionStatus(file))
           .setPreferredBlockSize(file.getPreferredBlockSize())
           .setReplication(file.getFileReplication())
           .setStoragePolicyID(file.getLocalStoragePolicyID());
 
       AclFeature f = file.getAclFeature();
       if (f != null) {
-        b.setAcl(buildAclEntries(f, state.getStringMap()));
+        b.setAcl(buildAclEntries(f));
       }
       XAttrFeature xAttrFeature = file.getXAttrFeature();
       if (xAttrFeature != null) {
-        b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap()));
+        b.setXAttrs(buildXAttrs(xAttrFeature));
       }
       return b;
     }
@@ -500,7 +451,7 @@ public final class FSImageFormatPBINode {
           .newBuilder().setModificationTime(dir.getModificationTime())
           .setNsQuota(quota.getNameSpace())
           .setDsQuota(quota.getStorageSpace())
-          .setPermission(buildPermissionStatus(dir, state.getStringMap()));
+          .setPermission(buildPermissionStatus(dir));
 
       if (quota.getTypeSpaces().anyGreaterOrEqual(0)) {
         b.setTypeQuotas(buildQuotaByStorageTypeEntries(quota));
@@ -508,11 +459,11 @@ public final class FSImageFormatPBINode {
 
       AclFeature f = dir.getAclFeature();
       if (f != null) {
-        b.setAcl(buildAclEntries(f, state.getStringMap()));
+        b.setAcl(buildAclEntries(f));
       }
       XAttrFeature xAttrFeature = dir.getXAttrFeature();
       if (xAttrFeature != null) {
-        b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap()));
+        b.setXAttrs(buildXAttrs(xAttrFeature));
       }
       return b;
     }
@@ -672,7 +623,7 @@ public final class FSImageFormatPBINode {
       SaverContext state = parent.getSaverContext();
       INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink
           .newBuilder()
-          .setPermission(buildPermissionStatus(n, state.getStringMap()))
+          .setPermission(buildPermissionStatus(n))
           .setTarget(ByteString.copyFrom(n.getSymlink()))
           .setModificationTime(n.getModificationTime())
           .setAccessTime(n.getAccessTime());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 6a7b9db..d5c99cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -80,10 +80,10 @@ public final class FSImageFormatProtobuf {
       .getLogger(FSImageFormatProtobuf.class);
 
   public static final class LoaderContext {
-    private String[] stringTable;
+    private SerialNumberManager.StringTable stringTable;
     private final ArrayList<INodeReference> refList = Lists.newArrayList();
 
-    public String[] getStringTable() {
+    public SerialNumberManager.StringTable getStringTable() {
       return stringTable;
     }
 
@@ -124,13 +124,6 @@ public final class FSImageFormatProtobuf {
     }
     private final ArrayList<INodeReference> refList = Lists.newArrayList();
 
-    private final DeduplicationMap<String> stringMap = DeduplicationMap
-        .newMap();
-
-    public DeduplicationMap<String> getStringMap() {
-      return stringMap;
-    }
-
     public ArrayList<INodeReference> getRefList() {
       return refList;
     }
@@ -312,11 +305,12 @@ public final class FSImageFormatProtobuf {
 
     private void loadStringTableSection(InputStream in) throws IOException {
       StringTableSection s = StringTableSection.parseDelimitedFrom(in);
-      ctx.stringTable = new String[s.getNumEntry() + 1];
+      ctx.stringTable =
+          SerialNumberManager.newStringTable(s.getNumEntry(), s.getMaskBits());
       for (int i = 0; i < s.getNumEntry(); ++i) {
         StringTableSection.Entry e = StringTableSection.Entry
             .parseDelimitedFrom(in);
-        ctx.stringTable[e.getId()] = e.getStr();
+        ctx.stringTable.put(e.getId(), e.getStr());
       }
     }
 
@@ -601,12 +595,16 @@ public final class FSImageFormatProtobuf {
     private void saveStringTableSection(FileSummary.Builder summary)
         throws IOException {
       OutputStream out = sectionOutputStream;
+
+      SerialNumberManager.StringTable stringTable =
+          SerialNumberManager.getStringTable();
       StringTableSection.Builder b = StringTableSection.newBuilder()
-          .setNumEntry(saverContext.stringMap.size());
+          .setNumEntry(stringTable.size())
+          .setMaskBits(stringTable.getMaskBits());
       b.build().writeDelimitedTo(out);
-      for (Entry<String, Integer> e : saverContext.stringMap.entrySet()) {
+      for (Entry<Integer, String> e : stringTable) {
         StringTableSection.Entry.Builder eb = StringTableSection.Entry
-            .newBuilder().setId(e.getValue()).setStr(e.getKey());
+            .newBuilder().setId(e.getKey()).setStr(e.getValue());
         eb.build().writeDelimitedTo(out);
       }
       commitSection(summary, SectionName.STRING_TABLE);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
index 48633f1..b7d2f2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
@@ -33,7 +33,9 @@ import com.google.common.base.Preconditions;
 @InterfaceAudience.Private
 public abstract class INodeWithAdditionalFields extends INode
     implements LinkedElement {
-  static enum PermissionStatusFormat {
+  // Note: this format is used both in-memory and on-disk.  Changes will be
+  // incompatible.
+  enum PermissionStatusFormat implements LongBitFormat.Enum {
     MODE(null, 16),
     GROUP(MODE.BITS, 24),
     USER(GROUP.BITS, 24);
@@ -46,12 +48,14 @@ public abstract class INodeWithAdditionalFields extends INode
 
     static String getUser(long permission) {
       final int n = (int)USER.BITS.retrieve(permission);
-      return SerialNumberManager.INSTANCE.getUser(n);
+      String s = SerialNumberManager.USER.getString(n);
+      assert s != null;
+      return s;
     }
 
     static String getGroup(long permission) {
       final int n = (int)GROUP.BITS.retrieve(permission);
-      return SerialNumberManager.INSTANCE.getGroup(n);
+      return SerialNumberManager.GROUP.getString(n);
     }
     
     static short getMode(long permission) {
@@ -61,16 +65,34 @@ public abstract class INodeWithAdditionalFields extends INode
     /** Encode the {@link PermissionStatus} to a long. */
     static long toLong(PermissionStatus ps) {
       long permission = 0L;
-      final int user = SerialNumberManager.INSTANCE.getUserSerialNumber(
+      final int user = SerialNumberManager.USER.getSerialNumber(
           ps.getUserName());
+      assert user != 0;
       permission = USER.BITS.combine(user, permission);
-      final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber(
+      // ideally should assert on group but inodes are created with null
+      // group and then updated only when added to a directory.
+      final int group = SerialNumberManager.GROUP.getSerialNumber(
           ps.getGroupName());
       permission = GROUP.BITS.combine(group, permission);
       final int mode = ps.getPermission().toShort();
       permission = MODE.BITS.combine(mode, permission);
       return permission;
     }
+
+    static PermissionStatus toPermissionStatus(long id,
+        SerialNumberManager.StringTable stringTable) {
+      int uid = (int)USER.BITS.retrieve(id);
+      int gid = (int)GROUP.BITS.retrieve(id);
+      return new PermissionStatus(
+          SerialNumberManager.USER.getString(uid, stringTable),
+          SerialNumberManager.GROUP.getString(gid, stringTable),
+          new FsPermission(getMode(id)));
+    }
+
+    @Override
+    public int getLength() {
+      return BITS.getLength();
+    }
   }
 
   /** The inode id. */
@@ -175,7 +197,7 @@ public abstract class INodeWithAdditionalFields extends INode
 
   @Override
   final void setUser(String user) {
-    int n = SerialNumberManager.INSTANCE.getUserSerialNumber(user);
+    int n = SerialNumberManager.USER.getSerialNumber(user);
     updatePermissionStatus(PermissionStatusFormat.USER, n);
   }
 
@@ -189,7 +211,7 @@ public abstract class INodeWithAdditionalFields extends INode
 
   @Override
   final void setGroup(String group) {
-    int n = SerialNumberManager.INSTANCE.getGroupSerialNumber(group);
+    int n = SerialNumberManager.GROUP.getSerialNumber(group);
     updatePermissionStatus(PermissionStatusFormat.GROUP, n);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
index eb497ab..927cc1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java
@@ -17,23 +17,195 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-/** Manage name-to-serial-number maps for users and groups. */
-class SerialNumberManager {
-  /** This is the only instance of {@link SerialNumberManager}.*/
-  static final SerialNumberManager INSTANCE = new SerialNumberManager();
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields.PermissionStatusFormat;
+import org.apache.hadoop.hdfs.util.LongBitFormat;
 
-  private final SerialNumberMap<String> usermap = new SerialNumberMap<String>();
-  private final SerialNumberMap<String> groupmap = new SerialNumberMap<String>();
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
 
-  private SerialNumberManager() {}
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_EXPANDED_STRING_TABLES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_IMAGE_EXPANDED_STRING_TABLES_KEY;
 
-  int getUserSerialNumber(String u) {return usermap.get(u);}
-  int getGroupSerialNumber(String g) {return groupmap.get(g);}
-  String getUser(int n) {return usermap.get(n);}
-  String getGroup(int n) {return groupmap.get(n);}
+/** Manage name-to-serial-number maps for various string tables. */
+public enum SerialNumberManager {
+  GLOBAL(), // NEVER EVER directly access!
+  USER(PermissionStatusFormat.USER, AclEntryStatusFormat.NAME),
+  GROUP(PermissionStatusFormat.GROUP, AclEntryStatusFormat.NAME),
+  XATTR(XAttrFormat.NAME);
 
-  {
-    getUserSerialNumber(null);
-    getGroupSerialNumber(null);
+  private static final SerialNumberManager[] values = values();
+  private static final int maxEntryBits;
+  private static final int maxEntryNumber;
+  private static final int maskBits;
+  private static boolean initialized;
+
+  private SerialNumberMap<String> serialMap;
+  private int bitLength = Integer.SIZE;
+  private boolean enabled;
+
+  static {
+    maxEntryBits = Integer.numberOfLeadingZeros(values.length);
+    maxEntryNumber = (1 << maxEntryBits) - 1;
+    maskBits = Integer.SIZE - maxEntryBits;
+    for (SerialNumberManager snm : values) {
+      // account for string table mask bits.
+      snm.updateLength(maxEntryBits);
+      // find max allowed length in case global is enabled.
+      GLOBAL.updateLength(snm.getLength());
+    }
+    // can reinitialize once later.
+    initializeSerialMaps(DFS_IMAGE_EXPANDED_STRING_TABLES_DEFAULT);
+  }
+
+  static synchronized void initialize(Configuration conf) {
+    boolean useExpanded = conf.getBoolean(
+        DFS_IMAGE_EXPANDED_STRING_TABLES_KEY,
+        DFS_IMAGE_EXPANDED_STRING_TABLES_DEFAULT);
+    if (initialized) {
+      if (useExpanded ^ !GLOBAL.enabled) {
+        throw new IllegalStateException("Cannot change serial maps");
+      }
+      return;
+    }
+    initializeSerialMaps(useExpanded);
+    for (SerialNumberManager snm : values) {
+      if (snm.enabled) {
+        FSDirectory.LOG.info(snm + " serial map: bits=" + snm.getLength() +
+            " maxEntries=" + snm.serialMap.getMax());
+      }
+    }
+    initialized = true;
+  }
+
+  private static void initializeSerialMaps(boolean useExpanded) {
+    if (useExpanded) {
+      // initialize per-manager serial maps for all but global.
+      for (SerialNumberManager snm : values) {
+        snm.enabled = (snm != GLOBAL);
+        snm.serialMap = snm.enabled ? new SerialNumberMap<String>(snm) : null;
+      }
+    } else {
+      // initialize all managers to use the global serial map.
+      SerialNumberMap<String> globalSerialMap = new SerialNumberMap<>(GLOBAL);
+      for (SerialNumberManager snm : values) {
+        snm.enabled = (snm == GLOBAL);
+        snm.serialMap = globalSerialMap;
+      }
+    }
+  }
+
+  SerialNumberManager(LongBitFormat.Enum... elements) {
+    // compute the smallest bit length registered with the serial manager.
+    for (LongBitFormat.Enum element : elements) {
+      updateLength(element.getLength());
+    }
+  }
+
+  int getLength() {
+    return bitLength;
+  }
+
+  private void updateLength(int maxLength) {
+    bitLength = Math.min(bitLength, maxLength);
+  }
+
+  public int getSerialNumber(String u) {
+    return serialMap.get(u);
+  }
+
+  public String getString(int id) {
+    return serialMap.get(id);
+  }
+
+  public String getString(int id, StringTable stringTable) {
+    return (stringTable != null)
+        ? stringTable.get(this, id) : getString(id);
+  }
+
+  private int getMask(int bits) {
+    return ordinal() << (Integer.SIZE - bits);
+  }
+
+  private static int getMaskBits() {
+    return GLOBAL.enabled ? 0 : maskBits;
+  }
+
+  private int size() {
+    return enabled ? serialMap.size() : 0;
+  }
+
+  private Iterable<Entry<Integer, String>> entrySet() {
+    if (!enabled) {
+      return Collections.emptySet();
+    }
+    return serialMap.entrySet();
+  }
+
+  // returns snapshot of current values for a save.
+  public static StringTable getStringTable() {
+    // approximate size for capacity.
+    int size = 0;
+    for (final SerialNumberManager snm : values) {
+      size += snm.size();
+    }
+    int tableMaskBits = getMaskBits();
+    StringTable map = new StringTable(size, tableMaskBits);
+    for (final SerialNumberManager snm : values) {
+      final int mask = snm.getMask(tableMaskBits);
+      for (Entry<Integer, String> entry : snm.entrySet()) {
+        map.put(entry.getKey() | mask, entry.getValue());
+      }
+    }
+    return map;
+  }
+
+  // returns an empty table for load.
+  public static StringTable newStringTable(int size, int bits) {
+    if (bits > maskBits) {
+      throw new IllegalArgumentException(
+        "String table bits " + bits + " > " + maskBits);
+    }
+    return new StringTable(size, bits);
+  }
+
+  public static class StringTable implements Iterable<Entry<Integer, String>> {
+    private final int tableMaskBits;
+    private final Map<Integer,String> map;
+
+    private StringTable(int size, int loadingMaskBits) {
+      this.tableMaskBits = loadingMaskBits;
+      this.map = new HashMap<>(size);
+    }
+
+    private String get(SerialNumberManager snm, int id) {
+      if (tableMaskBits != 0) {
+        if (id > maxEntryNumber) {
+          throw new IllegalStateException(
+              "serial id " + id + " > " + maxEntryNumber);
+        }
+        id |= snm.getMask(tableMaskBits);
+      }
+      return map.get(id);
+    }
+
+    public void put(int id, String str) {
+      map.put(id, str);
+    }
+
+    public Iterator<Entry<Integer, String>> iterator() {
+      return map.entrySet().iterator();
+    }
+
+    public int size() {
+      return map.size();
+    }
+
+    public int getMaskBits() {
+      return tableMaskBits;
+    }
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
index 2965837..ee6b8eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java
@@ -17,6 +17,10 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -35,21 +39,33 @@ import org.apache.hadoop.classification.InterfaceAudience;
  */
 @InterfaceAudience.Private
 public class SerialNumberMap<T> {
-  private final AtomicInteger max = new AtomicInteger(1);
+  private String name;
+  private final int max;
+  private final AtomicInteger current = new AtomicInteger(1);
   private final ConcurrentMap<T, Integer> t2i =
       new ConcurrentHashMap<T, Integer>();
   private final ConcurrentMap<Integer, T> i2t =
       new ConcurrentHashMap<Integer, T>();
 
+  SerialNumberMap(SerialNumberManager snm) {
+    this(snm.name(), snm.getLength());
+  }
+
+  SerialNumberMap(String name, int bitLength) {
+    this.name = name;
+    this.max = (1 << bitLength) - 1;
+  }
+
   public int get(T t) {
     if (t == null) {
       return 0;
     }
     Integer sn = t2i.get(t);
     if (sn == null) {
-      sn = max.getAndIncrement();
-      if (sn < 0) {
-        throw new IllegalStateException("Too many elements!");
+      sn = current.getAndIncrement();
+      if (sn > max) {
+        current.getAndDecrement();
+        throw new IllegalStateException(name + ": serial number map is full");
       }
       Integer old = t2i.putIfAbsent(t, sn);
       if (old != null) {
@@ -66,14 +82,27 @@ public class SerialNumberMap<T> {
     }
     T t = i2t.get(i);
     if (t == null) {
-      throw new IllegalStateException("!i2t.containsKey(" + i
-          + "), this=" + this);
+      throw new IllegalStateException(
+          name + ": serial number " + i + " does not exist");
     }
     return t;
   }
 
+  int getMax() {
+    return max;
+  }
+
+  Set<Map.Entry<Integer, T>> entrySet() {
+    return new HashSet<>(i2t.entrySet());
+  }
+
+  public int size() {
+    return i2t.size();
+  }
+
   @Override
   public String toString() {
-    return "max=" + max + ",\n  t2i=" + t2i + ",\n  i2t=" + i2t;
+    return "current=" + current + ",\n" +
+           "max=" + max + ",\n  t2i=" + t2i + ",\n  i2t=" + i2t;
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
index f9f06db..5b5992c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java
@@ -31,52 +31,66 @@ import org.apache.hadoop.hdfs.util.LongBitFormat;
 
 /**
  * Class to pack XAttrs into byte[].<br>
- * For each XAttr:<br>
- *   The first 4 bytes represents XAttr namespace and name<br>
- *     [0:3)  - XAttr namespace<br>
- *     [3:8) - Reserved<br>
- *     [8:32) - The name of the entry, which is an ID that points to a
- *              string in map<br>
- *   The following two bytes represents the length of XAttr value<br>
- *   The remaining bytes is the XAttr value<br>
+ *
+ * Note:  this format is used both in-memory and on-disk.  Changes will be
+ * incompatible.
+ *
  */
-class XAttrFormat {
-  private enum XAttrStatusFormat {
 
-    NAMESPACE(null, 3),
-    RESERVED(NAMESPACE.BITS, 5),
-    NAME(RESERVED.BITS, 24);
+public enum XAttrFormat implements LongBitFormat.Enum {
+  RESERVED(null, 5),
+  NS_EXT(RESERVED.BITS, 1),
+  NAME(NS_EXT.BITS, 24),
+  NS(NAME.BITS, 2);
 
-    private final LongBitFormat BITS;
+  private static final int NS_EXT_SHIFT = NS.BITS.getLength();
+  private static final int NS_MASK = (1 << NS_EXT_SHIFT) - 1;
 
-    XAttrStatusFormat(LongBitFormat previous, int length) {
-      BITS = new LongBitFormat(name(), previous, length, 0);
-    }
+  private static final int XATTR_VALUE_LEN_MAX = 1 << 16;
+  private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES =
+      XAttr.NameSpace.values();
 
-    static XAttr.NameSpace getNamespace(int xattrStatus) {
-      int ordinal = (int) NAMESPACE.BITS.retrieve(xattrStatus);
-      return XAttr.NameSpace.values()[ordinal];
-    }
+  private final LongBitFormat BITS;
 
-    static String getName(int xattrStatus) {
-      int id = (int) NAME.BITS.retrieve(xattrStatus);
-      return XAttrStorage.getName(id);
-    }
+  XAttrFormat(LongBitFormat previous, int length) {
+    BITS = new LongBitFormat(name(), previous, length, 0);
+  }
 
-    static int toInt(XAttr.NameSpace namespace, String name) {
-      long xattrStatusInt = 0;
+  @Override
+  public int getLength() {
+    return BITS.getLength();
+  }
 
-      xattrStatusInt = NAMESPACE.BITS
-          .combine(namespace.ordinal(), xattrStatusInt);
-      int nid = XAttrStorage.getNameSerialNumber(name);
-      xattrStatusInt = NAME.BITS
-          .combine(nid, xattrStatusInt);
+  static XAttr.NameSpace getNamespace(int record) {
+    long nid = NS.BITS.retrieve(record);
+    nid |= NS_EXT.BITS.retrieve(record) << NS_EXT_SHIFT;
+    return XATTR_NAMESPACE_VALUES[(int) nid];
+  }
 
-      return (int) xattrStatusInt;
-    }
+  public static String getName(int record) {
+    int nid = (int)NAME.BITS.retrieve(record);
+    return SerialNumberManager.XATTR.getString(nid);
   }
 
-  private static final int XATTR_VALUE_LEN_MAX = 1 << 16;
+  static int toInt(XAttr a) {
+    int nid = SerialNumberManager.XATTR.getSerialNumber(a.getName());
+    int nsOrd = a.getNameSpace().ordinal();
+    long value = NS.BITS.combine(nsOrd & NS_MASK, 0L);
+    value = NS_EXT.BITS.combine(nsOrd >>> NS_EXT_SHIFT, value);
+    value = NAME.BITS.combine(nid, value);
+    return (int)value;
+  }
+
+  static XAttr toXAttr(int record, byte[] value,
+                       SerialNumberManager.StringTable stringTable) {
+    int nid = (int)NAME.BITS.retrieve(record);
+    String name = SerialNumberManager.XATTR.getString(nid, stringTable);
+    return new XAttr.Builder()
+        .setNameSpace(getNamespace(record))
+        .setName(name)
+        .setValue(value)
+        .build();
+  }
 
   /**
    * Unpack byte[] to XAttrs.
@@ -95,8 +109,8 @@ class XAttrFormat {
       int v = Ints.fromBytes(attrs[i], attrs[i + 1],
           attrs[i + 2], attrs[i + 3]);
       i += 4;
-      builder.setNameSpace(XAttrStatusFormat.getNamespace(v));
-      builder.setName(XAttrStatusFormat.getName(v));
+      builder.setNameSpace(XAttrFormat.getNamespace(v));
+      builder.setName(XAttrFormat.getName(v));
       int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]);
       i += 2;
       if (vlen > 0) {
@@ -129,8 +143,8 @@ class XAttrFormat {
       int v = Ints.fromBytes(attrs[i], attrs[i + 1],
           attrs[i + 2], attrs[i + 3]);
       i += 4;
-      XAttr.NameSpace namespace = XAttrStatusFormat.getNamespace(v);
-      String name = XAttrStatusFormat.getName(v);
+      XAttr.NameSpace namespace = XAttrFormat.getNamespace(v);
+      String name = XAttrFormat.getName(v);
       int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]);
       i += 2;
       if (xAttr.getNameSpace() == namespace &&
@@ -161,7 +175,8 @@ class XAttrFormat {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     try {
       for (XAttr a : xAttrs) {
-        int v = XAttrStatusFormat.toInt(a.getNameSpace(), a.getName());
+        // big-endian
+        int v = XAttrFormat.toInt(a);
         out.write(Ints.toByteArray(v));
         int vlen = a.getValue() == null ? 0 : a.getValue().length;
         Preconditions.checkArgument(vlen < XATTR_VALUE_LEN_MAX,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
index 8a91e2a..a0abf6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
@@ -31,17 +31,6 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
 @InterfaceAudience.Private
 public class XAttrStorage {
 
-  private static final SerialNumberMap<String> NAME_MAP =
-      new SerialNumberMap<>();
-
-  public static int getNameSerialNumber(String name) {
-    return NAME_MAP.get(name);
-  }
-
-  public static String getName(int n) {
-    return NAME_MAP.get(n);
-  }
-
   /**
    * Reads the extended attribute of an inode by name with prefix.
    * <p/>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 21e9d2e..ca44982 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager;
 import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam;
 import org.apache.hadoop.io.IOUtils;
@@ -66,7 +67,7 @@ import com.google.common.collect.Maps;
 class FSImageLoader {
   public static final Log LOG = LogFactory.getLog(FSImageHandler.class);
 
-  private final String[] stringTable;
+  private final SerialNumberManager.StringTable stringTable;
   // byte representation of inodes, sorted by id
   private final byte[][] inodes;
   private final Map<Long, long[]> dirmap;
@@ -92,8 +93,8 @@ class FSImageLoader {
     }
   };
 
-  private FSImageLoader(String[] stringTable, byte[][] inodes,
-                        Map<Long, long[]> dirmap) {
+  private FSImageLoader(SerialNumberManager.StringTable stringTable,
+                        byte[][] inodes, Map<Long, long[]> dirmap) {
     this.stringTable = stringTable;
     this.inodes = inodes;
     this.dirmap = dirmap;
@@ -118,7 +119,7 @@ class FSImageLoader {
     try (FileInputStream fin = new FileInputStream(file.getFD())) {
       // Map to record INodeReference to the referred id
       ImmutableList<Long> refIdList = null;
-      String[] stringTable = null;
+      SerialNumberManager.StringTable stringTable = null;
       byte[][] inodes = null;
       Map<Long, long[]> dirmap = null;
 
@@ -241,16 +242,17 @@ class FSImageLoader {
     return inodes;
   }
 
-  static String[] loadStringTable(InputStream in) throws
-  IOException {
+  static SerialNumberManager.StringTable loadStringTable(InputStream in)
+        throws IOException {
     FsImageProto.StringTableSection s = FsImageProto.StringTableSection
         .parseDelimitedFrom(in);
     LOG.info("Loading " + s.getNumEntry() + " strings");
-    String[] stringTable = new String[s.getNumEntry() + 1];
+    SerialNumberManager.StringTable stringTable =
+        SerialNumberManager.newStringTable(s.getNumEntry(), s.getMaskBits());
     for (int i = 0; i < s.getNumEntry(); ++i) {
       FsImageProto.StringTableSection.Entry e = FsImageProto
           .StringTableSection.Entry.parseDelimitedFrom(in);
-      stringTable[e.getId()] = e.getStr();
+      stringTable.put(e.getId(), e.getStr());
     }
     return stringTable;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
index ae1cd79..5a1644c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import org.apache.hadoop.util.Time;
@@ -391,7 +392,7 @@ abstract class PBImageTextWriter implements Closeable {
     }
   }
 
-  private String[] stringTable;
+  private SerialNumberManager.StringTable stringTable;
   private PrintStream out;
   private MetadataMap metadataMap = null;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index 9ec97a6..e9c160a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -57,7 +57,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.NameSystemSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
-import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
+import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager;
 import org.apache.hadoop.hdfs.util.XMLUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import com.google.common.collect.ImmutableList;
@@ -229,7 +229,7 @@ public final class PBImageXmlWriter {
   private final Configuration conf;
   private final PrintStream out;
   private final SimpleDateFormat isoDateFormat;
-  private String[] stringTable;
+  private SerialNumberManager.StringTable stringTable;
 
   public static SimpleDateFormat createSimpleDateFormat() {
     SimpleDateFormat format =
@@ -387,8 +387,9 @@ public final class PBImageXmlWriter {
           ((XATTR_NAMESPACE_EXT_MASK & (encodedName >> XATTR_NAMESPACE_EXT_OFFSET)) << 2);
       o(INODE_SECTION_NS, XAttrProtos.XAttrProto.
           XAttrNamespaceProto.valueOf(ns).toString());
-      o(SECTION_NAME,
-          stringTable[XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET)]);
+      o(SECTION_NAME, SerialNumberManager.XATTR.getString(
+          XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET),
+          stringTable));
       ByteString val = xattr.getValue();
       if (val.isValidUtf8()) {
         o(INODE_SECTION_VAL, val.toStringUtf8());
@@ -692,10 +693,9 @@ public final class PBImageXmlWriter {
               .o(SNAPSHOT_DIFF_SECTION_CHILDREN_SIZE, d.getChildrenSize())
               .o(SNAPSHOT_DIFF_SECTION_IS_SNAPSHOT_ROOT, d.getIsSnapshotRoot())
               .o(SECTION_NAME, d.getName().toStringUtf8());
-          INodeDirectory snapshotCopy = d.getSnapshotCopy();
-          if (snapshotCopy != null) {
+          if (d.hasSnapshotCopy()) {
             out.print("<" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">");
-            dumpINodeDirectory(snapshotCopy);
+            dumpINodeDirectory(d.getSnapshotCopy());
             out.print("</" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">\n");
           }
           o(SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE, d.getCreatedListSize());
@@ -759,13 +759,7 @@ public final class PBImageXmlWriter {
   }
 
   private void loadStringTable(InputStream in) throws IOException {
-    StringTableSection s = StringTableSection.parseDelimitedFrom(in);
-    stringTable = new String[s.getNumEntry() + 1];
-    for (int i = 0; i < s.getNumEntry(); ++i) {
-      StringTableSection.Entry e = StringTableSection.Entry
-          .parseDelimitedFrom(in);
-      stringTable[e.getId()] = e.getStr();
-    }
+    stringTable = FSImageLoader.loadStringTable(in);
   }
 
   private PBImageXmlWriter o(final String e, final Object v) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 2fe2991..5fb7c69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -308,6 +308,7 @@ message StringTableSection {
     optional string str = 2;
   }
   optional uint32 numEntry = 1;
+  optional uint32 maskBits = 2 [default = 0];
   // repeated Entry
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04ca639a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index d3b387f..9104e2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -57,12 +57,8 @@ public class TestCommitBlockSynchronization {
     // set file's parent as root and put the file to inodeMap, so
     // FSNamesystem's isFileDeleted() method will return false on this file
     if (file.getParent() == null) {
-      INodeDirectory mparent = mock(INodeDirectory.class);
-      INodeDirectory parent = new INodeDirectory(mparent.getId(), new byte[0],
-          mparent.getPermissionStatus(), mparent.getAccessTime());
-      parent.setLocalName(new byte[0]);
+      INodeDirectory parent = namesystem.getFSDirectory().getRoot();
       parent.addChild(file);
-      file.setParent(parent);
     }
     namesystem.dir.getINodeMap().put(file);
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[04/50] [abbrv] hadoop git commit: YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan.

Posted by jh...@apache.org.
YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan.

(cherry picked from commit edce866489d83744f3f47a3b884b0c6136885e4a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c102ebf5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c102ebf5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c102ebf5

Branch: refs/heads/branch-2-jhung-test
Commit: c102ebf5b04bef1e805b3799ea825d2e87c801ad
Parents: d5cb071
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Oct 10 09:32:17 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../scheduler/ClusterNodeTracker.java           | 11 +++++
 .../scheduler/capacity/CapacityScheduler.java   | 12 ++++-
 .../capacity/TestContainerAllocation.java       | 52 ++++++++++++++++++++
 3 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c102ebf5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 010e645..b23b2be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -228,6 +229,16 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
     }
   }
 
+  @VisibleForTesting
+  public void setForceConfiguredMaxAllocation(boolean flag) {
+    writeLock.lock();
+    try {
+      forceConfiguredMaxAllocation = flag;
+    } finally {
+      writeLock.unlock();
+    }
+  }
+
   private void updateMaxResources(SchedulerNode node, boolean add) {
     Resource totalResource = node.getTotalResource();
     writeLock.lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c102ebf5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index aaae86d..207e059 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2192,7 +2192,17 @@ public class CapacityScheduler extends
       LOG.error("queue " + queueName + " is not an leaf queue");
       return getMaximumResourceCapability();
     }
-    return ((LeafQueue)queue).getMaximumAllocation();
+
+    // queue.getMaxAllocation returns *configured* maximum allocation.
+    // getMaximumResourceCapability() returns maximum allocation considers
+    // per-node maximum resources. So return (component-wise) min of the two.
+
+    Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation();
+    Resource clusterMaxAllocationConsiderNodeMax =
+        getMaximumResourceCapability();
+
+    return Resources.componentwiseMin(queueMaxAllocation,
+        clusterMaxAllocationConsiderNodeMax);
   }
 
   private String handleMoveToPlanQueue(String targetQueueName) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c102ebf5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 25e2c06..c1fcda8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
 import org.apache.hadoop.yarn.server.api.ContainerType;
 import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -65,6 +66,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB;
 
 public class TestContainerAllocation {
 
@@ -1062,4 +1064,54 @@ public class TestContainerAllocation {
 
     rm1.close();
   }
+
+  @Test(timeout = 60000)
+  public void testContainerRejectionWhenAskBeyondDynamicMax()
+      throws Exception {
+    CapacitySchedulerConfiguration newConf =
+        (CapacitySchedulerConfiguration) TestUtils
+            .getConfigurationWithMultipleQueues(conf);
+    newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+        DominantResourceCalculator.class, ResourceCalculator.class);
+    newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+        + MAXIMUM_ALLOCATION_MB, "4096");
+
+    MockRM rm1 = new MockRM(newConf);
+    rm1.start();
+
+    // before any node registered or before registration timeout,
+    // submit an app beyond queue max leads to failure.
+    boolean submitFailed = false;
+    MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1);
+    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+    try {
+      am1.allocate("*", 5 * GB, 1, null);
+    } catch (InvalidResourceRequestException e) {
+      submitFailed = true;
+    }
+    Assert.assertTrue(submitFailed);
+
+    // Ask 4GB succeeded.
+    am1.allocate("*", 4 * GB, 1, null);
+
+    // Add a new node, now the cluster maximum should be refreshed to 3GB.
+    CapacityScheduler cs = (CapacityScheduler)rm1.getResourceScheduler();
+    cs.getNodeTracker().setForceConfiguredMaxAllocation(false);
+    rm1.registerNode("h2:1234", 3 * GB, 1);
+
+    // Now ask 4 GB will fail
+    submitFailed = false;
+    try {
+      am1.allocate("*", 4 * GB, 1, null);
+    } catch (InvalidResourceRequestException e) {
+      submitFailed = true;
+    }
+    Assert.assertTrue(submitFailed);
+
+    // But ask 3 GB succeeded.
+    am1.allocate("*", 3 * GB, 1, null);
+
+    rm1.close();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[29/50] [abbrv] hadoop git commit: HDFS-14048. DFSOutputStream close() throws exception on subsequent call after DataNode restart. Contributed by Erik Krogen.

Posted by jh...@apache.org.
HDFS-14048. DFSOutputStream close() throws exception on subsequent call after DataNode restart. Contributed by Erik Krogen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba89181b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba89181b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba89181b

Branch: refs/heads/branch-2-jhung-test
Commit: ba89181be6998dd55bec1f302d3fefd21f10afbc
Parents: a4044fd
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Nov 6 12:16:46 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java      | 5 +++--
 .../hadoop/hdfs/TestClientProtocolForPipelineRecovery.java      | 2 ++
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba89181b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 02fb44a..fb7568d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -290,7 +290,7 @@ class DataStreamer extends Daemon {
     }
     packets.clear();
   }
-  
+
   static class LastExceptionInStreamer {
     private IOException thrown;
 
@@ -1754,6 +1754,7 @@ class DataStreamer extends Daemon {
         blockStream = out;
         result =  true; // success
         errorState.reset();
+        lastException.clear();
         // remove all restarting nodes from failed nodes list
         failed.removeAll(restartingNodes);
         restartingNodes.clear();
@@ -1835,7 +1836,7 @@ class DataStreamer extends Daemon {
 
   protected LocatedBlock locateFollowingBlock(DatanodeInfo[] excluded,
       ExtendedBlock oldBlock) throws IOException {
-    final DfsClientConf conf = dfsClient.getConf(); 
+    final DfsClientConf conf = dfsClient.getConf();
     int retries = conf.getNumBlockWriteLocateFollowingRetry();
     long sleeptime = conf.getBlockWriteLocateFollowingInitialDelayMs();
     while (true) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba89181b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index f5582be..6c22569 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -436,6 +436,8 @@ public class TestClientProtocolForPipelineRecovery {
           0, out.getStreamer().getPipelineRecoveryCount());
       out.write(1);
       out.close();
+      // Ensure that subsequent closes are idempotent and do not throw errors
+      out.close();
     } finally {
       if (cluster != null) {
         cluster.shutdown();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[10/50] [abbrv] hadoop git commit: YARN-3879 [Storage implementation] Create HDFS backing storage implementation for ATS reads. Contributed by Abhishek Modi.

Posted by jh...@apache.org.
YARN-3879 [Storage implementation] Create HDFS backing storage implementation for ATS reads. Contributed by Abhishek Modi.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be3f13da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be3f13da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be3f13da

Branch: refs/heads/branch-2-jhung-test
Commit: be3f13da781511604e41d9720dfff32d8797de99
Parents: 1045b69
Author: Vrushali C <vr...@apache.org>
Authored: Thu Oct 11 21:23:34 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../storage/FileSystemTimelineReaderImpl.java   | 87 +++++++++++---------
 1 file changed, 50 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be3f13da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index a0ee2be..6260c75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
@@ -38,6 +37,11 @@ import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@@ -68,7 +72,9 @@ public class FileSystemTimelineReaderImpl extends AbstractService
   private static final Logger LOG =
       LoggerFactory.getLogger(FileSystemTimelineReaderImpl.class);
 
-  private String rootPath;
+  private FileSystem fs;
+  private Path rootPath;
+  private Path entitiesPath;
   private static final String ENTITIES_DIR = "entities";
 
   /** Default extension for output files. */
@@ -94,7 +100,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
 
   @VisibleForTesting
   String getRootPath() {
-    return rootPath;
+    return rootPath.toString();
   }
 
   private static ObjectMapper mapper;
@@ -162,12 +168,12 @@ public class FileSystemTimelineReaderImpl extends AbstractService
     if (clusterId == null || appId == null) {
       throw new IOException("Unable to get flow info");
     }
-    String appFlowMappingFile = rootPath + File.separator +  ENTITIES_DIR +
-        File.separator + clusterId + File.separator + APP_FLOW_MAPPING_FILE;
+    Path clusterIdPath = new Path(entitiesPath, clusterId);
+    Path appFlowMappingFilePath = new Path(clusterIdPath,
+            APP_FLOW_MAPPING_FILE);
     try (BufferedReader reader =
              new BufferedReader(new InputStreamReader(
-                 new FileInputStream(
-                     appFlowMappingFile), Charset.forName("UTF-8")));
+                 fs.open(appFlowMappingFilePath), Charset.forName("UTF-8")));
          CSVParser parser = new CSVParser(reader, csvFormat)) {
       for (CSVRecord record : parser.getRecords()) {
         if (record.size() < 4) {
@@ -266,7 +272,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
     return entity;
   }
 
-  private Set<TimelineEntity> getEntities(File dir, String entityType,
+  private Set<TimelineEntity> getEntities(Path dir, String entityType,
       TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
       throws IOException {
     // First sort the selected entities based on created/start time.
@@ -280,15 +286,18 @@ public class FileSystemTimelineReaderImpl extends AbstractService
           }
         );
     if (dir != null) {
-      File[] files = dir.listFiles();
-      if (files != null) {
-        for (File entityFile : files) {
+      RemoteIterator<LocatedFileStatus> fileStatuses = fs.listFiles(dir,
+              false);
+      if (fileStatuses != null) {
+        while (fileStatuses.hasNext()) {
+          LocatedFileStatus locatedFileStatus = fileStatuses.next();
+          Path entityFile = locatedFileStatus.getPath();
           if (!entityFile.getName()
               .contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
             continue;
           }
           try (BufferedReader reader = new BufferedReader(
-              new InputStreamReader(new FileInputStream(entityFile),
+              new InputStreamReader(fs.open(entityFile),
                   Charset.forName("UTF-8")))) {
             TimelineEntity entity = readEntityFromFile(reader);
             if (!entity.getType().equals(entityType)) {
@@ -366,25 +375,30 @@ public class FileSystemTimelineReaderImpl extends AbstractService
 
   @Override
   public void serviceInit(Configuration conf) throws Exception {
-    rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+    String outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
         conf.get("hadoop.tmp.dir") + File.separator + STORAGE_DIR_ROOT);
+    rootPath = new Path(outputRoot);
+    entitiesPath = new Path(rootPath, ENTITIES_DIR);
+    fs = rootPath.getFileSystem(conf);
     super.serviceInit(conf);
   }
 
   @Override
   public TimelineEntity getEntity(TimelineReaderContext context,
       TimelineDataToRetrieve dataToRetrieve) throws IOException {
-    String flowRunPath = getFlowRunPath(context.getUserId(),
+    String flowRunPathStr = getFlowRunPath(context.getUserId(),
         context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
         context.getAppId());
-    File dir = new File(new File(rootPath, ENTITIES_DIR),
-        context.getClusterId() + File.separator + flowRunPath + File.separator +
-        context.getAppId() + File.separator + context.getEntityType());
-    File entityFile = new File(
-        dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+    Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+    Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+    Path appIdPath = new Path(flowRunPath, context.getAppId());
+    Path entityTypePath = new Path(appIdPath, context.getEntityType());
+    Path entityFilePath = new Path(entityTypePath,
+            context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
+
     try (BufferedReader reader =
              new BufferedReader(new InputStreamReader(
-                 new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+                 fs.open(entityFilePath), Charset.forName("UTF-8")))) {
       TimelineEntity entity = readEntityFromFile(reader);
       return createEntityToBeReturned(
           entity, dataToRetrieve.getFieldsToRetrieve());
@@ -399,32 +413,31 @@ public class FileSystemTimelineReaderImpl extends AbstractService
   public Set<TimelineEntity> getEntities(TimelineReaderContext context,
       TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
       throws IOException {
-    String flowRunPath = getFlowRunPath(context.getUserId(),
+    String flowRunPathStr = getFlowRunPath(context.getUserId(),
         context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
         context.getAppId());
-    File dir =
-        new File(new File(rootPath, ENTITIES_DIR),
-            context.getClusterId() + File.separator + flowRunPath +
-            File.separator + context.getAppId() + File.separator +
-            context.getEntityType());
-    return getEntities(dir, context.getEntityType(), filters, dataToRetrieve);
+    Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+    Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+    Path appIdPath = new Path(flowRunPath, context.getAppId());
+    Path entityTypePath = new Path(appIdPath, context.getEntityType());
+
+    return getEntities(entityTypePath, context.getEntityType(), filters,
+            dataToRetrieve);
   }
 
   @Override public Set<String> getEntityTypes(TimelineReaderContext context)
       throws IOException {
     Set<String> result = new TreeSet<>();
-    String flowRunPath = getFlowRunPath(context.getUserId(),
+    String flowRunPathStr = getFlowRunPath(context.getUserId(),
         context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
         context.getAppId());
-    File dir = new File(new File(rootPath, ENTITIES_DIR),
-        context.getClusterId() + File.separator + flowRunPath
-            + File.separator + context.getAppId());
-    File[] fileList = dir.listFiles();
-    if (fileList != null) {
-      for (File f : fileList) {
-        if (f.isDirectory()) {
-          result.add(f.getName());
-        }
+    Path clusterIdPath = new Path(entitiesPath, context.getClusterId());
+    Path flowRunPath = new Path(clusterIdPath, flowRunPathStr);
+    Path appIdPath = new Path(flowRunPath, context.getAppId());
+    FileStatus[] fileStatuses = fs.listStatus(appIdPath);
+    for (FileStatus fileStatus : fileStatuses) {
+      if (fileStatus.isDirectory()) {
+        result.add(fileStatus.getPath().getName());
       }
     }
     return result;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[43/50] [abbrv] hadoop git commit: HDFS-13963. NN UI is broken with IE11. Contributed by Ayush Saxena.

Posted by jh...@apache.org.
HDFS-13963. NN UI is broken with IE11. Contributed by Ayush Saxena.

(cherry picked from commit 8571507efa3af3e4a5213a7e24c65e26162eac4d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cebbfd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cebbfd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cebbfd5

Branch: refs/heads/branch-2-jhung-test
Commit: 0cebbfd5703409fbc464cb8f8c5158ab9f288195
Parents: fc3e979
Author: Vinayakumar B <vi...@apache.org>
Authored: Fri Nov 16 17:26:36 2018 +0530
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html  | 2 +-
 .../hadoop-hdfs/src/main/webapps/datanode/datanode.html            | 2 +-
 .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html               | 2 +-
 .../hadoop-hdfs/src/main/webapps/hdfs/explorer.html                | 2 +-
 .../hadoop-hdfs/src/main/webapps/journal/index.html                | 2 +-
 .../hadoop-hdfs/src/main/webapps/secondary/status.html             | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
index cf19c94..553eba3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -18,7 +18,7 @@
 -->
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<meta http-equiv="X-UA-Compatible" content="IE=edge" />
 <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
 <link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />
 <link rel="stylesheet" type="text/css" href="/static/rbf.css" />

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
index 1bb37fb..08fd354 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html
@@ -18,7 +18,7 @@
 -->
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<meta http-equiv="X-UA-Compatible" content="IE=edge" />
 <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
 <link rel="stylesheet" type="text/css" href="/static/hadoop.css" />
 <title>DataNode Information</title>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 6ae3960..4bc10c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -18,7 +18,7 @@
 -->
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<meta http-equiv="X-UA-Compatible" content="IE=edge" />
 <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
 <link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />
 <link rel="stylesheet" type="text/css" href="/static/hadoop.css" />

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index 3700a5e..1370097 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -18,7 +18,7 @@
   -->
 <html xmlns="http://www.w3.org/1999/xhtml">
   <head>
-    <meta http-equiv="X-UA-Compatible" content="IE=9" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
     <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
     <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap-editable.css"/>
     <link rel="stylesheet" type="text/css" href="/static/dataTables.bootstrap.css" />

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/index.html
index 72ab2cc..f1fd4fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/index.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/index.html
@@ -18,7 +18,7 @@
 -->
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-<meta http-equiv="X-UA-Compatible" content="IE=9" />
+<meta http-equiv="X-UA-Compatible" content="IE=edge" />
 <link rel="stylesheet" type="text/css" href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
 <link rel="stylesheet" type="text/css" href="/static/hadoop.css" />
 <title>JournalNode Information</title>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cebbfd5/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
index 3db97df..4e488bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html
@@ -18,7 +18,7 @@
 -->
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  <meta http-equiv="X-UA-Compatible" content="IE=9" />
+  <meta http-equiv="X-UA-Compatible" content="IE=edge" />
   <link rel="stylesheet" type="text/css"
        href="/static/bootstrap-3.0.2/css/bootstrap.min.css" />
   <link rel="stylesheet" type="text/css" href="/static/hadoop.css" />


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[03/50] [abbrv] hadoop git commit: YARN-8893. [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client. Contributed by Botong Huang.

Posted by jh...@apache.org.
YARN-8893. [AMRMProxy] Fix thread leak in AMRMClientRelayer and UAM client. Contributed by Botong Huang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/690d8088
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/690d8088
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/690d8088

Branch: refs/heads/branch-2-jhung-test
Commit: 690d80887bf1a01ad633fe2d826a4d689b60cf78
Parents: 611b5c7
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 15:30:08 2018 -0700
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/server/AMRMClientRelayer.java   | 55 +++++---------------
 .../yarn/server/uam/UnmanagedAMPoolManager.java | 28 ++++++++++
 .../server/uam/UnmanagedApplicationManager.java | 28 +++++++---
 .../yarn/server/MockResourceManagerFacade.java  |  5 +-
 .../yarn/server/TestAMRMClientRelayer.java      | 10 ++--
 .../metrics/TestAMRMClientRelayerMetrics.java   |  6 ---
 .../uam/TestUnmanagedApplicationManager.java    | 27 +++++++++-
 .../amrmproxy/FederationInterceptor.java        | 18 +++----
 .../amrmproxy/TestAMRMProxyService.java         |  1 -
 9 files changed, 102 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
index 5c7f7a7..a01d8e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/AMRMClientRelayer.java
@@ -27,9 +27,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -46,8 +45,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
 import org.apache.hadoop.yarn.api.records.UpdatedContainer;
 import org.apache.hadoop.yarn.client.AMRMClientUtils;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
 import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -65,8 +62,7 @@ import com.google.common.annotations.VisibleForTesting;
  * pending requests similar to AMRMClient, and handles RM re-sync automatically
  * without propagate the re-sync exception back to AMRMClient.
  */
-public class AMRMClientRelayer extends AbstractService
-    implements ApplicationMasterProtocol {
+public class AMRMClientRelayer implements ApplicationMasterProtocol {
   private static final Logger LOG =
       LoggerFactory.getLogger(AMRMClientRelayer.class);
 
@@ -131,51 +127,16 @@ public class AMRMClientRelayer extends AbstractService
 
   private AMRMClientRelayerMetrics metrics;
 
-  public AMRMClientRelayer() {
-    super(AMRMClientRelayer.class.getName());
+  public AMRMClientRelayer(ApplicationMasterProtocol rmClient,
+      ApplicationId appId, String rmId) {
     this.resetResponseId = -1;
     this.metrics = AMRMClientRelayerMetrics.getInstance();
-    this.rmClient = null;
-    this.appId = null;
     this.rmId = "";
-  }
-
-  public AMRMClientRelayer(ApplicationMasterProtocol rmClient,
-      ApplicationId appId, String rmId) {
-    this();
     this.rmClient = rmClient;
     this.appId = appId;
     this.rmId = rmId;
   }
 
-  @Override
-  protected void serviceInit(Configuration conf) throws Exception {
-    super.serviceInit(conf);
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
-    final YarnConfiguration conf = new YarnConfiguration(getConfig());
-    try {
-      if (this.rmClient == null) {
-        this.rmClient =
-            ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
-      }
-    } catch (IOException e) {
-      throw new YarnRuntimeException(e);
-    }
-    super.serviceStart();
-  }
-
-  @Override
-  protected void serviceStop() throws Exception {
-    if (this.rmClient != null) {
-      RPC.stopProxy(this.rmClient);
-    }
-    shutdown();
-    super.serviceStop();
-  }
-
   public void setAMRegistrationRequest(
       RegisterApplicationMasterRequest registerRequest) {
     this.amRegistrationRequest = registerRequest;
@@ -226,6 +187,14 @@ public class AMRMClientRelayer extends AbstractService
             .decrClientPending(rmId, req.getContainerUpdateType(), 1);
       }
     }
+
+    if (this.rmClient != null) {
+      try {
+        RPC.stopProxy(this.rmClient);
+        this.rmClient = null;
+      } catch (HadoopIllegalArgumentException e) {
+      }
+    }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
index 10a1ddb..e36cc15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
@@ -371,6 +371,34 @@ public class UnmanagedAMPoolManager extends AbstractService {
   }
 
   /**
+   * Shutdown an UAM client without killing it in YarnRM.
+   *
+   * @param uamId uam Id
+   * @throws YarnException if fails
+   */
+  public void shutDownConnections(String uamId)
+      throws YarnException {
+    if (!this.unmanagedAppMasterMap.containsKey(uamId)) {
+      throw new YarnException("UAM " + uamId + " does not exist");
+    }
+    LOG.info(
+        "Shutting down UAM id {} for application {} without killing the UAM",
+        uamId, this.appIdMap.get(uamId));
+    this.unmanagedAppMasterMap.remove(uamId).shutDownConnections();
+  }
+
+  /**
+   * Shutdown all UAM clients without killing them in YarnRM.
+   *
+   * @throws YarnException if fails
+   */
+  public void shutDownConnections() throws YarnException {
+    for (String uamId : this.unmanagedAppMasterMap.keySet()) {
+      shutDownConnections(uamId);
+    }
+  }
+
+  /**
    * Get the id of all running UAMs.
    *
    * @return uamId set

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
index c762ebb..46abdc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
@@ -255,9 +255,6 @@ public class UnmanagedApplicationManager {
   public FinishApplicationMasterResponse finishApplicationMaster(
       FinishApplicationMasterRequest request)
       throws YarnException, IOException {
-
-    this.heartbeatHandler.shutdown();
-
     if (this.userUgi == null) {
       if (this.connectionInitiated) {
         // This is possible if the async launchUAM is still
@@ -270,7 +267,12 @@ public class UnmanagedApplicationManager {
             + "be called before createAndRegister");
       }
     }
-    return this.rmProxyRelayer.finishApplicationMaster(request);
+    FinishApplicationMasterResponse response =
+        this.rmProxyRelayer.finishApplicationMaster(request);
+    if (response.getIsUnregistered()) {
+      shutDownConnections();
+    }
+    return response;
   }
 
   /**
@@ -282,11 +284,10 @@ public class UnmanagedApplicationManager {
    */
   public KillApplicationResponse forceKillApplication()
       throws IOException, YarnException {
+    shutDownConnections();
+
     KillApplicationRequest request =
         KillApplicationRequest.newInstance(this.applicationId);
-
-    this.heartbeatHandler.shutdown();
-
     if (this.rmClient == null) {
       this.rmClient = createRMProxy(ApplicationClientProtocol.class, this.conf,
           UserGroupInformation.createRemoteUser(this.submitter), null);
@@ -324,6 +325,14 @@ public class UnmanagedApplicationManager {
   }
 
   /**
+   * Shutdown this UAM client, without killing the UAM in the YarnRM side.
+   */
+  public void shutDownConnections() {
+    this.heartbeatHandler.shutdown();
+    this.rmProxyRelayer.shutdown();
+  }
+
+  /**
    * Returns the application id of the UAM.
    *
    * @return application id of the UAM
@@ -532,4 +541,9 @@ public class UnmanagedApplicationManager {
   protected void drainHeartbeatThread() {
     this.heartbeatHandler.drainHeartbeatThread();
   }
+
+  @VisibleForTesting
+  protected boolean isHeartbeatThreadAlive() {
+    return this.heartbeatHandler.isAlive();
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 8dca937..df75f81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -108,7 +108,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
@@ -323,9 +322,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
       applicationContainerIdMap.remove(appId);
     }
 
-    return FinishApplicationMasterResponse.newInstance(
-        request.getFinalApplicationStatus() == FinalApplicationStatus.SUCCEEDED
-            ? true : false);
+    return FinishApplicationMasterResponse.newInstance(true);
   }
 
   protected ApplicationId getApplicationId(int id) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
index fa46960..46570a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/TestAMRMClientRelayer.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestExceptio
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.scheduler.ResourceRequestSet;
 import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -155,16 +156,17 @@ public class TestAMRMClientRelayer {
 
     this.mockAMS = new MockApplicationMasterService();
     this.relayer = new AMRMClientRelayer(this.mockAMS, null, "TEST");
-
-    this.relayer.init(conf);
-    this.relayer.start();
-
     this.relayer.registerApplicationMaster(
         RegisterApplicationMasterRequest.newInstance("", 0, ""));
 
     clearAllocateRequestLists();
   }
 
+  @After
+  public void cleanup() {
+    this.relayer.shutdown();
+  }
+
   private void assertAsksAndReleases(int expectedAsk, int expectedRelease) {
     Assert.assertEquals(expectedAsk, this.mockAMS.lastAsk.size());
     Assert.assertEquals(expectedRelease, this.mockAMS.lastRelease.size());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
index ebbfae2..dd48241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/metrics/TestAMRMClientRelayerMetrics.java
@@ -141,17 +141,11 @@ public class TestAMRMClientRelayerMetrics {
 
     this.homeRelayer = new AMRMClientRelayer(this.mockAMS,
         ApplicationId.newInstance(0, 0), this.homeID);
-    this.homeRelayer.init(conf);
-    this.homeRelayer.start();
-
     this.homeRelayer.registerApplicationMaster(
         RegisterApplicationMasterRequest.newInstance("", 0, ""));
 
     this.uamRelayer = new AMRMClientRelayer(this.mockAMS,
         ApplicationId.newInstance(0, 0), this.uamID);
-    this.uamRelayer.init(conf);
-    this.uamRelayer.start();
-
     this.uamRelayer.registerApplicationMaster(
         RegisterApplicationMasterRequest.newInstance("", 0, ""));
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
index 54e7dd3..abb1d93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
@@ -87,7 +87,7 @@ public class TestUnmanagedApplicationManager {
     }
   }
 
-  @Test(timeout = 5000)
+  @Test(timeout = 10000)
   public void testBasicUsage()
       throws YarnException, IOException, InterruptedException {
 
@@ -104,6 +104,11 @@ public class TestUnmanagedApplicationManager {
     finishApplicationMaster(
         FinishApplicationMasterRequest.newInstance(null, null, null),
         attemptId);
+
+    while (uam.isHeartbeatThreadAlive()) {
+      LOG.info("waiting for heartbeat thread to finish");
+      Thread.sleep(100);
+    }
   }
 
   /*
@@ -261,7 +266,7 @@ public class TestUnmanagedApplicationManager {
         attemptId);
   }
 
-  @Test
+  @Test(timeout = 10000)
   public void testForceKill()
       throws YarnException, IOException, InterruptedException {
     launchUAM(attemptId);
@@ -269,6 +274,11 @@ public class TestUnmanagedApplicationManager {
         RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
     uam.forceKillApplication();
 
+    while (uam.isHeartbeatThreadAlive()) {
+      LOG.info("waiting for heartbeat thread to finish");
+      Thread.sleep(100);
+    }
+
     try {
       uam.forceKillApplication();
       Assert.fail("Should fail because application is already killed");
@@ -276,6 +286,19 @@ public class TestUnmanagedApplicationManager {
     }
   }
 
+  @Test(timeout = 10000)
+  public void testShutDownConnections()
+      throws YarnException, IOException, InterruptedException {
+    launchUAM(attemptId);
+    registerApplicationMaster(
+        RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
+    uam.shutDownConnections();
+    while (uam.isHeartbeatThreadAlive()) {
+      LOG.info("waiting for heartbeat thread to finish");
+      Thread.sleep(100);
+    }
+  }
+
   protected UserGroupInformation getUGIWithToken(
       ApplicationAttemptId appAttemptId) {
     UserGroupInformation ugi =

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index c478871..ae9f78d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -716,12 +716,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
                   uamPool.finishApplicationMaster(subClusterId, finishRequest);
 
               if (uamResponse.getIsUnregistered()) {
-                AMRMClientRelayer relayer =
-                    secondaryRelayers.remove(subClusterId);
-                if(relayer != null) {
-                  relayer.shutdown();
-                }
-
+                secondaryRelayers.remove(subClusterId);
                 if (getNMStateStore() != null) {
                   getNMStateStore().removeAMRMProxyAppContextEntry(attemptId,
                       NMSS_SECONDARY_SC_PREFIX + subClusterId);
@@ -801,8 +796,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
    */
   @Override
   public void shutdown() {
+    LOG.info("Shutting down FederationInterceptor for {}", this.attemptId);
+
     // Do not stop uamPool service and kill UAMs here because of possible second
     // app attempt
+    try {
+      this.uamPool.shutDownConnections();
+    } catch (YarnException e) {
+      LOG.error("Error shutting down all UAM clients without killing them", e);
+    }
+
     if (this.threadpool != null) {
       try {
         this.threadpool.shutdown();
@@ -814,9 +817,6 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
     // Stop the home heartbeat thread
     this.homeHeartbeartHandler.shutdown();
     this.homeRMRelayer.shutdown();
-    for (AMRMClientRelayer relayer : this.secondaryRelayers.values()) {
-      relayer.shutdown();
-    }
 
     super.shutdown();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/690d8088/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 1eefbd5..b269fa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -206,7 +206,6 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
         finishApplicationMaster(testAppId, FinalApplicationStatus.FAILED);
 
     Assert.assertNotNull(finshResponse);
-    Assert.assertEquals(false, finshResponse.getIsUnregistered());
 
     try {
       // Try to finish an application master that is already finished.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[05/50] [abbrv] hadoop git commit: YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi.

Posted by jh...@apache.org.
YARN-8720. CapacityScheduler does not enforce max resource allocation check at queue level. Contributed by Tarun Parimi.

(cherry picked from commit f1a893fdbc2dbe949cae786f08bdb2651b88d673)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5cb0713
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5cb0713
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5cb0713

Branch: refs/heads/branch-2-jhung-test
Commit: d5cb07134d3dfafa64c7acc2f6fe61727e6ae059
Parents: 9f50f9c
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Sep 14 16:33:51 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java | 3 ++-
 .../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 +++--
 .../hadoop/yarn/server/resourcemanager/TestAppManager.java      | 3 +++
 .../hadoop/yarn/server/resourcemanager/TestClientRMService.java | 3 +++
 4 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5cb0713/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index eea42e7..273e0cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -207,7 +207,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
       }
     }
 
-    Resource maximumCapacity = getScheduler().getMaximumResourceCapability();
+    Resource maximumCapacity =
+        getScheduler().getMaximumResourceCapability(app.getQueue());
 
     // sanity check
     try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5cb0713/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 3b47db2..2ffdbc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -498,10 +498,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
         }
 
         // Normalize all requests
+        String queue = submissionContext.getQueue();
         for (ResourceRequest amReq : amReqs) {
           SchedulerUtils.normalizeAndValidateRequest(amReq,
-              scheduler.getMaximumResourceCapability(),
-              submissionContext.getQueue(), scheduler, isRecovery, rmContext);
+              scheduler.getMaximumResourceCapability(queue),
+              queue, scheduler, isRecovery, rmContext);
 
           amReq.setCapability(
               scheduler.getNormalizedResource(amReq.getCapability()));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5cb0713/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index adf6bbe..0623cb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -892,6 +892,9 @@ public class TestAppManager{
     when(scheduler.getMaximumResourceCapability()).thenReturn(
         Resources.createResource(
             YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+    when(scheduler.getMaximumResourceCapability(any(String.class))).thenReturn(
+        Resources.createResource(
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
 
     ResourceCalculator rs = mock(ResourceCalculator.class);
     when(scheduler.getResourceCalculator()).thenReturn(rs);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5cb0713/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 091bcd9..6d2e4f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1166,6 +1166,9 @@ public class TestClientRMService {
     when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
         Resources.createResource(
             YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+    when(yarnScheduler.getMaximumResourceCapability(any(String.class)))
+        .thenReturn(Resources.createResource(
+            YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
     when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
         Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
     when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[26/50] [abbrv] hadoop git commit: HDFS-13870. WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc. Contributed by Siyao Meng.

Posted by jh...@apache.org.
HDFS-13870. WebHDFS: Document ALLOWSNAPSHOT and DISALLOWSNAPSHOT API doc. Contributed by Siyao Meng.

(cherry picked from commit 0e36e935d909862401890d0a5410204504f48b31)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c206f18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c206f18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c206f18

Branch: refs/heads/branch-2-jhung-test
Commit: 2c206f189c07c78636f191b65ea5cac2692612c5
Parents: e320762
Author: Yiqun Lin <yq...@apache.org>
Authored: Fri Nov 30 11:31:34 2018 +0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop-hdfs/src/site/markdown/WebHDFS.md    | 24 ++++++++++++++++++++
 1 file changed, 24 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c206f18/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 3d7b32f..20dd8b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -62,6 +62,8 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
     * [`SETTIMES`](#Set_Access_or_Modification_Time) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setTimes)
     * [`RENEWDELEGATIONTOKEN`](#Renew_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).renewDelegationToken)
     * [`CANCELDELEGATIONTOKEN`](#Cancel_Delegation_Token) (see [DelegationTokenAuthenticator](../../api/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.html).cancelDelegationToken)
+    * [`ALLOWSNAPSHOT`](#Allow_Snapshot)
+    * [`DISALLOWSNAPSHOT`](#Disallow_Snapshot)
     * [`CREATESNAPSHOT`](#Create_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).createSnapshot)
     * [`RENAMESNAPSHOT`](#Rename_Snapshot) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).renameSnapshot)
     * [`SETXATTR`](#Set_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setXAttr)
@@ -1224,6 +1226,28 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listXAttr
 Snapshot Operations
 -------------------
 
+### Allow Snapshot
+
+* Submit a HTTP PUT request.
+
+        curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=ALLOWSNAPSHOT"
+
+    The client receives a response with zero content length on success:
+
+        HTTP/1.1 200 OK
+        Content-Length: 0
+
+### Disallow Snapshot
+
+* Submit a HTTP PUT request.
+
+        curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=DISALLOWSNAPSHOT"
+
+    The client receives a response with zero content length on success:
+
+        HTTP/1.1 200 OK
+        Content-Length: 0
+
 ### Create Snapshot
 
 * Submit a HTTP PUT request.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[28/50] [abbrv] hadoop git commit: HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"

Posted by jh...@apache.org.
 HADOOP-15959. Revert "HADOOP-12751. While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple"


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7144eefd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7144eefd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7144eefd

Branch: refs/heads/branch-2-jhung-test
Commit: 7144eefd1b11afcdf7f62d71fbac43080a94097a
Parents: 2c206f1
Author: Steve Loughran <st...@apache.org>
Authored: Fri Nov 30 11:33:19 2018 +0000
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../authentication/util/KerberosName.java       |  9 ++--
 .../TestKerberosAuthenticationHandler.java      |  7 ++-
 .../authentication/util/TestKerberosName.java   | 17 ++++++--
 .../java/org/apache/hadoop/security/KDiag.java  | 46 +-------------------
 .../src/site/markdown/SecureMode.md             |  6 ---
 .../org/apache/hadoop/security/TestKDiag.java   | 16 -------
 .../security/TestUserGroupInformation.java      | 27 ++++--------
 7 files changed, 33 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
index 5f50573..5eafd48 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java
@@ -324,8 +324,8 @@ public class KerberosName {
         }
       }
       if (result != null && nonSimplePattern.matcher(result).find()) {
-        LOG.info("Non-simple name {} after auth_to_local rule {}",
-            result, this);
+        throw new NoMatchingRule("Non-simple name " + result +
+                                 " after auth_to_local rule " + this);
       }
       if (toLowerCase && result != null) {
         result = result.toLowerCase(Locale.ENGLISH);
@@ -378,7 +378,7 @@ public class KerberosName {
   /**
    * Get the translation of the principal name into an operating system
    * user name.
-   * @return the user name
+   * @return the short name
    * @throws IOException throws if something is wrong with the rules
    */
   public String getShortName() throws IOException {
@@ -398,8 +398,7 @@ public class KerberosName {
         return result;
       }
     }
-    LOG.info("No auth_to_local rules applied to {}", this);
-    return toString();
+    throw new NoMatchingRule("No rules applied to " + toString());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
index e3444ef..408563f 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java
@@ -109,7 +109,12 @@ public class TestKerberosAuthenticationHandler
     kn = new KerberosName("bar@BAR");
     Assert.assertEquals("bar", kn.getShortName());
     kn = new KerberosName("bar@FOO");
-    Assert.assertEquals("bar@FOO", kn.getShortName());
+    try {
+      kn.getShortName();
+      Assert.fail();
+    }
+    catch (Exception ex) {      
+    }
   }
 
   @Test(timeout=60000)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
index c584fce..2db0df4 100644
--- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
+++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java
@@ -72,14 +72,23 @@ public class TestKerberosName {
     }
   }
 
+  private void checkBadTranslation(String from) {
+    System.out.println("Checking bad translation for " + from);
+    KerberosName nm = new KerberosName(from);
+    try {
+      nm.getShortName();
+      Assert.fail("didn't get exception for " + from);
+    } catch (IOException ie) {
+      // PASS
+    }
+  }
+
   @Test
   public void testAntiPatterns() throws Exception {
     checkBadName("owen/owen/owen@FOO.COM");
     checkBadName("owen@foo/bar.com");
-
-    // no rules applied, these should pass
-    checkTranslation("foo@ACME.COM", "foo@ACME.COM");
-    checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM");
+    checkBadTranslation("foo@ACME.COM");
+    checkBadTranslation("root/joe@FOO.COM");
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
index d71269d..f75e474 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java
@@ -25,7 +25,6 @@ import org.apache.directory.shared.kerberos.components.EncryptionKey;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.util.ExitUtil;
@@ -54,7 +53,6 @@ import java.util.Collections;
 import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
 import static org.apache.hadoop.security.UserGroupInformation.*;
@@ -130,12 +128,6 @@ public class KDiag extends Configured implements Tool, Closeable {
   private boolean nofail = false;
   private boolean nologin = false;
   private boolean jaas = false;
-  private boolean checkShortName = false;
-
-  /**
-   * A pattern that recognizes simple/non-simple names. Per KerberosName
-   */
-  private static final Pattern nonSimplePattern = Pattern.compile("[/@]");
 
   /**
    * Flag set to true if a {@link #verify(boolean, String, String, Object...)}
@@ -164,8 +156,6 @@ public class KDiag extends Configured implements Tool, Closeable {
 
   public static final String ARG_SECURE = "--secure";
 
-  public static final String ARG_VERIFYSHORTNAME = "--verifyshortname";
-
   @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
   public KDiag(Configuration conf,
       PrintWriter out,
@@ -209,7 +199,6 @@ public class KDiag extends Configured implements Tool, Closeable {
     nofail = popOption(ARG_NOFAIL, args);
     jaas = popOption(ARG_JAAS, args);
     nologin = popOption(ARG_NOLOGIN, args);
-    checkShortName = popOption(ARG_VERIFYSHORTNAME, args);
 
     // look for list of resources
     String resource;
@@ -255,9 +244,7 @@ public class KDiag extends Configured implements Tool, Closeable {
       + arg(ARG_NOLOGIN, "", "Do not attempt to log in")
       + arg(ARG_OUTPUT, "<file>", "Write output to a file")
       + arg(ARG_RESOURCE, "<resource>", "Load an XML configuration resource")
-      + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure")
-      + arg(ARG_VERIFYSHORTNAME, ARG_PRINCIPAL + " <principal>",
-      "Verify the short name of the specific principal does not contain '@' or '/'");
+      + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure");
   }
 
   private String arg(String name, String params, String meaning) {
@@ -290,7 +277,6 @@ public class KDiag extends Configured implements Tool, Closeable {
     println("%s = %d", ARG_KEYLEN, minKeyLength);
     println("%s = %s", ARG_KEYTAB, keytab);
     println("%s = %s", ARG_PRINCIPAL, principal);
-    println("%s = %s", ARG_VERIFYSHORTNAME, checkShortName);
 
     // Fail fast on a JVM without JCE installed.
     validateKeyLength();
@@ -390,10 +376,6 @@ public class KDiag extends Configured implements Tool, Closeable {
       validateJAAS(jaas);
       validateNTPConf();
 
-      if (checkShortName) {
-        validateShortName();
-      }
-
       if (!nologin) {
         title("Logging in");
         if (keytab != null) {
@@ -448,32 +430,6 @@ public class KDiag extends Configured implements Tool, Closeable {
   }
 
   /**
-   * Verify whether auth_to_local rules transform a principal name
-   * <p>
-   * Having a local user name "bar@foo.com" may be harmless, so it is noted at
-   * info. However if what was intended is a transformation to "bar"
-   * it can be difficult to debug, hence this check.
-   */
-  protected void validateShortName() {
-    failif(principal == null, CAT_KERBEROS, "No principal defined");
-
-    try {
-      KerberosName kn = new KerberosName(principal);
-      String result = kn.getShortName();
-      if (nonSimplePattern.matcher(result).find()) {
-        warn(CAT_KERBEROS, principal + " short name: " + result
-                + " still contains @ or /");
-      }
-    } catch (IOException e) {
-      throw new KerberosDiagsFailure(CAT_KERBEROS, e,
-              "Failed to get short name for " + principal, e);
-    } catch (IllegalArgumentException e) {
-      error(CAT_KERBEROS, "KerberosName(" + principal + ") failed: %s\n%s",
-              e, StringUtils.stringifyException(e));
-    }
-  }
-
-  /**
    * Get the default realm.
    * <p>
    * Not having a default realm may be harmless, so is noted at info.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index dc2d48f..d206b53 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -476,7 +476,6 @@ KDiag: Diagnose Kerberos Problems
   [--out <file>] : Write output to a file.
   [--resource <resource>] : Load an XML configuration resource.
   [--secure] : Require the hadoop configuration to be secure.
-  [--verifyshortname <principal>]: Verify the short name of the specific principal does not contain '@' or '/'
 ```
 
 #### `--jaas`: Require a JAAS file to be defined in `java.security.auth.login.config`.
@@ -575,11 +574,6 @@ or implicitly set to "simple":
 
 Needless to say, an application so configured cannot talk to a secure Hadoop cluster.
 
-#### `--verifyshortname <principal>`: validate the short name of a principal
-
-This verifies that the short name of a principal contains neither the `"@"`
-nor `"/"` characters.
-
 ### Example
 
 ```

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
index b81c4c5..1ba11cc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java
@@ -165,22 +165,6 @@ public class TestKDiag extends Assert {
   }
 
   @Test
-  public void testKerberosName() throws Throwable {
-    kdiagFailure(ARG_KEYLEN, KEYLEN,
-            ARG_VERIFYSHORTNAME,
-            ARG_PRINCIPAL, "foo/foo/foo@BAR.COM");
-  }
-
-  @Test
-  public void testShortName() throws Throwable {
-    kdiag(ARG_KEYLEN, KEYLEN,
-            ARG_KEYTAB, keytab.getAbsolutePath(),
-            ARG_PRINCIPAL,
-            ARG_VERIFYSHORTNAME,
-            ARG_PRINCIPAL, "foo@EXAMPLE.COM");
-  }
-
-  @Test
   public void testFileOutput() throws Throwable {
     File f = new File("target/kdiag.txt");
     kdiag(ARG_KEYLEN, KEYLEN,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7144eefd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index a6b9823..2974f3e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -297,15 +297,10 @@ public class TestUserGroupInformation {
     UserGroupInformation.setConfiguration(conf);
     testConstructorSuccess("user1", "user1");
     testConstructorSuccess("user4@OTHER.REALM", "other-user4");
-
-    // pass through test, no transformation
-    testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM");
-    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM");
-    testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
-    // failures
-    testConstructorFailures("user6@example.com@OTHER.REALM");
-    testConstructorFailures("user7@example.com@DEFAULT.REALM");
+    // failure test
+    testConstructorFailures("user2@DEFAULT.REALM");
+    testConstructorFailures("user3/cron@DEFAULT.REALM");
+    testConstructorFailures("user5/cron@OTHER.REALM");
     testConstructorFailures(null);
     testConstructorFailures("");
   }
@@ -319,13 +314,10 @@ public class TestUserGroupInformation {
 
     testConstructorSuccess("user1", "user1");
     testConstructorSuccess("user2@DEFAULT.REALM", "user2");
-    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");
-
-    // no rules applied, local name remains the same
-    testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM");
-    testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM");
-
+    testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3");    
     // failure test
+    testConstructorFailures("user4@OTHER.REALM");
+    testConstructorFailures("user5/cron@OTHER.REALM");
     testConstructorFailures(null);
     testConstructorFailures("");
   }
@@ -366,9 +358,8 @@ public class TestUserGroupInformation {
     } catch (IllegalArgumentException e) {
       String expect = (userName == null || userName.isEmpty())
           ? "Null user" : "Illegal principal name "+userName;
-      String expect2 = "Malformed Kerberos name: "+userName;
-      assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e,
-          e.toString().contains(expect) || e.toString().contains(expect2));
+      assertTrue("Did not find "+ expect + " in " + e,
+          e.toString().contains(expect));
     }
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[11/50] [abbrv] hadoop git commit: HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended.

Posted by jh...@apache.org.
HADOOP-15679. ShutdownHookManager shutdown time needs to be configurable & extended.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/079aedbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/079aedbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/079aedbe

Branch: refs/heads/branch-2-jhung-test
Commit: 079aedbeb68063efc0ab63d4139f2c6495750552
Parents: c6cc685
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 12:43:09 2018 +0100
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:55 2018 -0800

----------------------------------------------------------------------
 .../fs/CommonConfigurationKeysPublic.java       |   9 +
 .../apache/hadoop/util/ShutdownHookManager.java | 169 ++++++++--
 .../src/main/resources/core-default.xml         |  16 +
 .../hadoop/util/TestShutdownHookManager.java    | 328 +++++++++++++++----
 4 files changed, 418 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/079aedbe/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index a2f88ac..df4dfbe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -910,5 +910,14 @@ public class CommonConfigurationKeysPublic {
           "credential$",
           "oauth.*token$",
           HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS));
+
+  /** Configuration option for the shutdown hook manager shutdown time:
+   *  {@value}. */
+  public static final String SERVICE_SHUTDOWN_TIMEOUT =
+      "hadoop.service.shutdown.timeout";
+
+  /** Default shutdown hook timeout: {@value} seconds. */
+  public static final long SERVICE_SHUTDOWN_TIMEOUT_DEFAULT = 30;
+
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/079aedbe/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
index 41907fb..8831f33 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java
@@ -17,10 +17,16 @@
  */
 package org.apache.hadoop.util;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -34,6 +40,9 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT_DEFAULT;
+
 /**
  * The <code>ShutdownHookManager</code> enables running shutdownHook
  * in a deterministic order, higher priority first.
@@ -42,53 +51,55 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * This class registers a single JVM shutdownHook and run all the
  * shutdownHooks registered to it (to this class) in order based on their
  * priority.
+ *
+ * Unless a hook was registered with a shutdown explicitly set through
+ * {@link #addShutdownHook(Runnable, int, long, TimeUnit)},
+ * the shutdown time allocated to it is set by the configuration option
+ * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in
+ * {@code core-site.xml}, with a default value of
+ * {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT}
+ * seconds.
  */
-public class ShutdownHookManager {
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class ShutdownHookManager {
 
   private static final ShutdownHookManager MGR = new ShutdownHookManager();
 
   private static final Logger LOG =
       LoggerFactory.getLogger(ShutdownHookManager.class);
-  private static final long TIMEOUT_DEFAULT = 10;
-  private static final TimeUnit TIME_UNIT_DEFAULT = TimeUnit.SECONDS;
+
+  /** Minimum shutdown timeout: {@value} second(s). */
+  public static final long TIMEOUT_MINIMUM = 1;
+
+  /** The default time unit used: seconds. */
+  public static final TimeUnit TIME_UNIT_DEFAULT = TimeUnit.SECONDS;
 
   private static final ExecutorService EXECUTOR =
       Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
-          .setDaemon(true).build());
+          .setDaemon(true)
+          .setNameFormat("shutdown-hook-%01d")
+          .build());
+
   static {
     try {
       Runtime.getRuntime().addShutdownHook(
         new Thread() {
           @Override
           public void run() {
-            MGR.shutdownInProgress.set(true);
-            for (HookEntry entry: MGR.getShutdownHooksInOrder()) {
-              Future<?> future = EXECUTOR.submit(entry.getHook());
-              try {
-                future.get(entry.getTimeout(), entry.getTimeUnit());
-              } catch (TimeoutException ex) {
-                future.cancel(true);
-                LOG.warn("ShutdownHook '" + entry.getHook().getClass().
-                    getSimpleName() + "' timeout, " + ex.toString(), ex);
-              } catch (Throwable ex) {
-                LOG.warn("ShutdownHook '" + entry.getHook().getClass().
-                    getSimpleName() + "' failed, " + ex.toString(), ex);
-              }
-            }
-            try {
-              EXECUTOR.shutdown();
-              if (!EXECUTOR.awaitTermination(TIMEOUT_DEFAULT,
-                  TIME_UNIT_DEFAULT)) {
-                LOG.error("ShutdownHookManger shutdown forcefully.");
-                EXECUTOR.shutdownNow();
-              }
-              LOG.debug("ShutdownHookManger complete shutdown.");
-            } catch (InterruptedException ex) {
-              LOG.error("ShutdownHookManger interrupted while waiting for " +
-                  "termination.", ex);
-              EXECUTOR.shutdownNow();
-              Thread.currentThread().interrupt();
+            if (MGR.shutdownInProgress.getAndSet(true)) {
+              LOG.info("Shutdown process invoked a second time: ignoring");
+              return;
             }
+            long started = System.currentTimeMillis();
+            int timeoutCount = executeShutdown();
+            long ended = System.currentTimeMillis();
+            LOG.debug(String.format(
+                "Completed shutdown in %.3f seconds; Timeouts: %d",
+                (ended-started)/1000.0, timeoutCount));
+            // each of the hooks have executed; now shut down the
+            // executor itself.
+            shutdownExecutor(new Configuration());
           }
         }
       );
@@ -99,18 +110,92 @@ public class ShutdownHookManager {
   }
 
   /**
+   * Execute the shutdown.
+   * This is exposed purely for testing: do not invoke it.
+   * @return the number of shutdown hooks which timed out.
+   */
+  @InterfaceAudience.Private
+  @VisibleForTesting
+  static int executeShutdown() {
+    int timeouts = 0;
+    for (HookEntry entry: MGR.getShutdownHooksInOrder()) {
+      Future<?> future = EXECUTOR.submit(entry.getHook());
+      try {
+        future.get(entry.getTimeout(), entry.getTimeUnit());
+      } catch (TimeoutException ex) {
+        timeouts++;
+        future.cancel(true);
+        LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+            getSimpleName() + "' timeout, " + ex.toString(), ex);
+      } catch (Throwable ex) {
+        LOG.warn("ShutdownHook '" + entry.getHook().getClass().
+            getSimpleName() + "' failed, " + ex.toString(), ex);
+      }
+    }
+    return timeouts;
+  }
+
+  /**
+   * Shutdown the executor thread itself.
+   * @param conf the configuration containing the shutdown timeout setting.
+   */
+  private static void shutdownExecutor(final Configuration conf) {
+    try {
+      EXECUTOR.shutdown();
+      long shutdownTimeout = getShutdownTimeout(conf);
+      if (!EXECUTOR.awaitTermination(
+          shutdownTimeout,
+          TIME_UNIT_DEFAULT)) {
+        // timeout waiting for the
+        LOG.error("ShutdownHookManger shutdown forcefully after"
+            + " {} seconds.", shutdownTimeout);
+        EXECUTOR.shutdownNow();
+      }
+      LOG.debug("ShutdownHookManger completed shutdown.");
+    } catch (InterruptedException ex) {
+      // interrupted.
+      LOG.error("ShutdownHookManger interrupted while waiting for " +
+          "termination.", ex);
+      EXECUTOR.shutdownNow();
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /**
    * Return <code>ShutdownHookManager</code> singleton.
    *
    * @return <code>ShutdownHookManager</code> singleton.
    */
+  @InterfaceAudience.Public
   public static ShutdownHookManager get() {
     return MGR;
   }
 
   /**
+   * Get the shutdown timeout in seconds, from the supplied
+   * configuration.
+   * @param conf configuration to use.
+   * @return a timeout, always greater than or equal to {@link #TIMEOUT_MINIMUM}
+   */
+  @InterfaceAudience.Private
+  @VisibleForTesting
+  static long getShutdownTimeout(Configuration conf) {
+    long duration = conf.getTimeDuration(
+        SERVICE_SHUTDOWN_TIMEOUT,
+        SERVICE_SHUTDOWN_TIMEOUT_DEFAULT,
+        TIME_UNIT_DEFAULT);
+    if (duration < TIMEOUT_MINIMUM) {
+      duration = TIMEOUT_MINIMUM;
+    }
+    return duration;
+  }
+
+  /**
    * Private structure to store ShutdownHook, its priority and timeout
    * settings.
    */
+  @InterfaceAudience.Private
+  @VisibleForTesting
   static class HookEntry {
     private final Runnable hook;
     private final int priority;
@@ -118,7 +203,9 @@ public class ShutdownHookManager {
     private final TimeUnit unit;
 
     HookEntry(Runnable hook, int priority) {
-      this(hook, priority, TIMEOUT_DEFAULT, TIME_UNIT_DEFAULT);
+      this(hook, priority,
+          getShutdownTimeout(new Configuration()),
+          TIME_UNIT_DEFAULT);
     }
 
     HookEntry(Runnable hook, int priority, long timeout, TimeUnit unit) {
@@ -176,10 +263,12 @@ public class ShutdownHookManager {
    *
    * @return the list of shutdownHooks in order of execution.
    */
+  @InterfaceAudience.Private
+  @VisibleForTesting
   List<HookEntry> getShutdownHooksInOrder() {
     List<HookEntry> list;
     synchronized (MGR.hooks) {
-      list = new ArrayList<HookEntry>(MGR.hooks);
+      list = new ArrayList<>(MGR.hooks);
     }
     Collections.sort(list, new Comparator<HookEntry>() {
 
@@ -200,6 +289,8 @@ public class ShutdownHookManager {
    * @param shutdownHook shutdownHook <code>Runnable</code>
    * @param priority priority of the shutdownHook.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public void addShutdownHook(Runnable shutdownHook, int priority) {
     if (shutdownHook == null) {
       throw new IllegalArgumentException("shutdownHook cannot be NULL");
@@ -223,6 +314,8 @@ public class ShutdownHookManager {
    * @param timeout timeout of the shutdownHook
    * @param unit unit of the timeout <code>TimeUnit</code>
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public void addShutdownHook(Runnable shutdownHook, int priority, long timeout,
       TimeUnit unit) {
     if (shutdownHook == null) {
@@ -242,6 +335,8 @@ public class ShutdownHookManager {
    * @return TRUE if the shutdownHook was registered and removed,
    * FALSE otherwise.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public boolean removeShutdownHook(Runnable shutdownHook) {
     if (shutdownInProgress.get()) {
       throw new IllegalStateException("Shutdown in progress, cannot remove a " +
@@ -256,6 +351,8 @@ public class ShutdownHookManager {
    * @param shutdownHook shutdownHook to check if registered.
    * @return TRUE/FALSE depending if the shutdownHook is is registered.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public boolean hasShutdownHook(Runnable shutdownHook) {
     return hooks.contains(new HookEntry(shutdownHook, 0));
   }
@@ -265,6 +362,8 @@ public class ShutdownHookManager {
    * 
    * @return TRUE if the shutdown is in progress, otherwise FALSE.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public boolean isShutdownInProgress() {
     return shutdownInProgress.get();
   }
@@ -272,7 +371,9 @@ public class ShutdownHookManager {
   /**
    * clear all registered shutdownHooks.
    */
+  @InterfaceAudience.Public
+  @InterfaceStability.Stable
   public void clearShutdownHooks() {
     hooks.clear();
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/079aedbe/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8415ae7..e262b33 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -562,6 +562,22 @@
     </description>
 </property>
 
+  <property>
+    <name>hadoop.service.shutdown.timeout</name>
+    <value>30s</value>
+    <description>
+      Timeout to wait for each shutdown operation to complete.
+      If a hook takes longer than this time to complete, it will be interrupted,
+      so the service will shutdown. This allows the service shutdown
+      to recover from a blocked operation.
+      Some shutdown hooks may need more time than this, for example when
+      a large amount of data needs to be uploaded to an object store.
+      In this situation: increase the timeout.
+
+      The minimum duration of the timeout is 1 second, "1s".
+    </description>
+</property>
+
 <property>
   <name>hadoop.rpc.protection</name>
   <value>authentication</value>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/079aedbe/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
index 2aa5e95..03fa903 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
@@ -17,97 +17,285 @@
  */
 package org.apache.hadoop.util;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.slf4j.LoggerFactory;
-import org.junit.Assert;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
 import org.junit.Test;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
 
 import static java.lang.Thread.sleep;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.SERVICE_SHUTDOWN_TIMEOUT_DEFAULT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 public class TestShutdownHookManager {
+
   static final Logger LOG =
       LoggerFactory.getLogger(TestShutdownHookManager.class.getName());
 
+  /**
+   * remove all the shutdown hooks so that they never get invoked later
+   * on in this test process.
+   */
+  @After
+  public void clearShutdownHooks() {
+    ShutdownHookManager.get().clearShutdownHooks();
+  }
+
+  /**
+   * Verify hook registration, then execute the hook callback stage
+   * of shutdown to verify invocation, execution order and timeout
+   * processing.
+   */
   @Test
   public void shutdownHookManager() {
     ShutdownHookManager mgr = ShutdownHookManager.get();
-    Assert.assertNotNull(mgr);
-    Assert.assertEquals(0, mgr.getShutdownHooksInOrder().size());
-    Runnable hook1 = new Runnable() {
-      @Override
-      public void run() {
-        LOG.info("Shutdown hook1 complete.");
-      }
-    };
-    Runnable hook2 = new Runnable() {
-      @Override
-      public void run() {
-        LOG.info("Shutdown hook2 complete.");
-      }
-    };
-
-    Runnable hook3 = new Runnable() {
-      @Override
-      public void run() {
-        try {
-          sleep(3000);
-          LOG.info("Shutdown hook3 complete.");
-        } catch (InterruptedException ex) {
-          LOG.info("Shutdown hook3 interrupted exception:",
-              ExceptionUtils.getStackTrace(ex));
-          Assert.fail("Hook 3 should not timeout.");
-        }
-      }
-    };
-
-    Runnable hook4 = new Runnable() {
-      @Override
-      public void run() {
-        try {
-          sleep(3500);
-          LOG.info("Shutdown hook4 complete.");
-          Assert.fail("Hook 4 should timeout");
-        } catch (InterruptedException ex) {
-          LOG.info("Shutdown hook4 interrupted exception:",
-              ExceptionUtils.getStackTrace(ex));
-        }
-      }
-    };
+    assertNotNull("No ShutdownHookManager", mgr);
+    assertEquals(0, mgr.getShutdownHooksInOrder().size());
+    Hook hook1 = new Hook("hook1", 0, false);
+    Hook hook2 = new Hook("hook2", 0, false);
+    Hook hook3 = new Hook("hook3", 1000, false);
+    Hook hook4 = new Hook("hook4", 25000, true);
+    Hook hook5 = new Hook("hook5",
+        (SERVICE_SHUTDOWN_TIMEOUT_DEFAULT + 1) * 1000, true);
 
     mgr.addShutdownHook(hook1, 0);
-    Assert.assertTrue(mgr.hasShutdownHook(hook1));
-    Assert.assertEquals(1, mgr.getShutdownHooksInOrder().size());
-    Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(0).getHook());
-    mgr.removeShutdownHook(hook1);
-    Assert.assertFalse(mgr.hasShutdownHook(hook1));
+    assertTrue(mgr.hasShutdownHook(hook1));
+    assertEquals(1, mgr.getShutdownHooksInOrder().size());
+    assertEquals(hook1, mgr.getShutdownHooksInOrder().get(0).getHook());
+    assertTrue(mgr.removeShutdownHook(hook1));
+    assertFalse(mgr.hasShutdownHook(hook1));
+    assertFalse(mgr.removeShutdownHook(hook1));
 
     mgr.addShutdownHook(hook1, 0);
-    Assert.assertTrue(mgr.hasShutdownHook(hook1));
-    Assert.assertEquals(1, mgr.getShutdownHooksInOrder().size());
-    Assert.assertTrue(mgr.hasShutdownHook(hook1));
-    Assert.assertEquals(1, mgr.getShutdownHooksInOrder().size());
+    assertTrue(mgr.hasShutdownHook(hook1));
+    assertEquals(1, mgr.getShutdownHooksInOrder().size());
+    assertEquals(SERVICE_SHUTDOWN_TIMEOUT_DEFAULT,
+        mgr.getShutdownHooksInOrder().get(0).getTimeout());
 
     mgr.addShutdownHook(hook2, 1);
-    Assert.assertTrue(mgr.hasShutdownHook(hook1));
-    Assert.assertTrue(mgr.hasShutdownHook(hook2));
-    Assert.assertEquals(2, mgr.getShutdownHooksInOrder().size());
-    Assert.assertEquals(hook2, mgr.getShutdownHooksInOrder().get(0).getHook());
-    Assert.assertEquals(hook1, mgr.getShutdownHooksInOrder().get(1).getHook());
+    assertTrue(mgr.hasShutdownHook(hook1));
+    assertTrue(mgr.hasShutdownHook(hook2));
+    assertEquals(2, mgr.getShutdownHooksInOrder().size());
+    assertEquals(hook2, mgr.getShutdownHooksInOrder().get(0).getHook());
+    assertEquals(hook1, mgr.getShutdownHooksInOrder().get(1).getHook());
 
     // Test hook finish without timeout
     mgr.addShutdownHook(hook3, 2, 4, TimeUnit.SECONDS);
-    Assert.assertTrue(mgr.hasShutdownHook(hook3));
-    Assert.assertEquals(hook3, mgr.getShutdownHooksInOrder().get(0).getHook());
-    Assert.assertEquals(4, mgr.getShutdownHooksInOrder().get(0).getTimeout());
-
-    // Test hook finish with timeout
-    mgr.addShutdownHook(hook4, 3, 2, TimeUnit.SECONDS);
-    Assert.assertTrue(mgr.hasShutdownHook(hook4));
-    Assert.assertEquals(hook4, mgr.getShutdownHooksInOrder().get(0).getHook());
-    Assert.assertEquals(2, mgr.getShutdownHooksInOrder().get(0).getTimeout());
-    LOG.info("Shutdown starts here");
+    assertTrue(mgr.hasShutdownHook(hook3));
+    assertEquals(hook3, mgr.getShutdownHooksInOrder().get(0).getHook());
+    assertEquals(4, mgr.getShutdownHooksInOrder().get(0).getTimeout());
+
+    // Test hook finish with timeout; highest priority
+    int hook4timeout = 2;
+    mgr.addShutdownHook(hook4, 3, hook4timeout, TimeUnit.SECONDS);
+    assertTrue(mgr.hasShutdownHook(hook4));
+    assertEquals(hook4, mgr.getShutdownHooksInOrder().get(0).getHook());
+    assertEquals(2, mgr.getShutdownHooksInOrder().get(0).getTimeout());
+
+    // a default timeout hook and verify it gets the default timeout
+    mgr.addShutdownHook(hook5, 5);
+    ShutdownHookManager.HookEntry hookEntry5 = mgr.getShutdownHooksInOrder()
+        .get(0);
+    assertEquals(hook5, hookEntry5.getHook());
+    assertEquals("default timeout not used",
+        ShutdownHookManager.getShutdownTimeout(new Configuration()),
+        hookEntry5.getTimeout());
+    assertEquals("hook priority", 5, hookEntry5.getPriority());
+    // remove this to avoid a longer sleep in the test run
+    assertTrue("failed to remove " + hook5,
+        mgr.removeShutdownHook(hook5));
+
+
+    // now execute the hook shutdown sequence
+    INVOCATION_COUNT.set(0);
+    LOG.info("invoking executeShutdown()");
+    int timeouts = ShutdownHookManager.executeShutdown();
+    LOG.info("Shutdown completed");
+    assertEquals("Number of timed out hooks", 1, timeouts);
+
+    List<ShutdownHookManager.HookEntry> hooks
+        = mgr.getShutdownHooksInOrder();
+
+    // analyze the hooks
+    for (ShutdownHookManager.HookEntry entry : hooks) {
+      Hook hook = (Hook) entry.getHook();
+      assertTrue("Was not invoked " + hook, hook.invoked);
+      // did any hook raise an exception?
+      hook.maybeThrowAssertion();
+    }
+
+    // check the state of some of the invoked hooks
+    // hook4 was invoked first, but it timed out.
+    assertEquals("Expected to be invoked first " + hook4,
+        1, hook4.invokedOrder);
+    assertFalse("Expected to time out " + hook4, hook4.completed);
+
+
+    // hook1 completed, but in order after the others, so its start time
+    // is the longest.
+    assertTrue("Expected to complete " + hook1, hook1.completed);
+    long invocationInterval = hook1.startTime - hook4.startTime;
+    assertTrue("invocation difference too short " + invocationInterval,
+        invocationInterval >= hook4timeout * 1000);
+    assertTrue("sleeping hook4 blocked other threads for " + invocationInterval,
+        invocationInterval < hook4.sleepTime);
+
+    // finally, clear the hooks
+    mgr.clearShutdownHooks();
+    // and verify that the hooks are empty
+    assertFalse(mgr.hasShutdownHook(hook1));
+    assertEquals("shutdown hook list is not empty",
+        0,
+        mgr.getShutdownHooksInOrder().size());
+  }
+
+  @Test
+  public void testShutdownTimeoutConfiguration() throws Throwable {
+    // set the shutdown timeout and verify it can be read back.
+    Configuration conf = new Configuration();
+    long shutdownTimeout = 5;
+    conf.setTimeDuration(SERVICE_SHUTDOWN_TIMEOUT,
+        shutdownTimeout, TimeUnit.SECONDS);
+    assertEquals(SERVICE_SHUTDOWN_TIMEOUT,
+        shutdownTimeout,
+        ShutdownHookManager.getShutdownTimeout(conf));
+  }
+
+  /**
+   * Verify that low timeouts simply fall back to
+   * {@link ShutdownHookManager#TIMEOUT_MINIMUM}.
+   */
+  @Test
+  public void testShutdownTimeoutBadConfiguration() throws Throwable {
+    // set the shutdown timeout and verify it can be read back.
+    Configuration conf = new Configuration();
+    long shutdownTimeout = 50;
+    conf.setTimeDuration(SERVICE_SHUTDOWN_TIMEOUT,
+        shutdownTimeout, TimeUnit.NANOSECONDS);
+    assertEquals(SERVICE_SHUTDOWN_TIMEOUT,
+        ShutdownHookManager.TIMEOUT_MINIMUM,
+        ShutdownHookManager.getShutdownTimeout(conf));
+  }
+
+  /**
+   * Verifies that a hook cannot be re-registered: an attempt to do so
+   * will simply be ignored.
+   */
+  @Test
+  public void testDuplicateRegistration() throws Throwable {
+    ShutdownHookManager mgr = ShutdownHookManager.get();
+    Hook hook = new Hook("hook1", 0, false);
+
+    // add the hook
+    mgr.addShutdownHook(hook, 2, 1, TimeUnit.SECONDS);
+
+    // add it at a higher priority. This will be ignored.
+    mgr.addShutdownHook(hook, 5);
+    List<ShutdownHookManager.HookEntry> hookList
+        = mgr.getShutdownHooksInOrder();
+    assertEquals("Hook added twice", 1, hookList.size());
+    ShutdownHookManager.HookEntry entry = hookList.get(0);
+    assertEquals("priority of hook", 2, entry.getPriority());
+    assertEquals("timeout of hook", 1, entry.getTimeout());
+
+    // remove the hook
+    assertTrue("failed to remove hook " + hook, mgr.removeShutdownHook(hook));
+    // which will fail a second time
+    assertFalse("expected hook removal to fail", mgr.removeShutdownHook(hook));
+
+    // now register it
+    mgr.addShutdownHook(hook, 5);
+    hookList = mgr.getShutdownHooksInOrder();
+    entry = hookList.get(0);
+    assertEquals("priority of hook", 5, entry.getPriority());
+    assertNotEquals("timeout of hook", 1, entry.getTimeout());
+
+  }
+
+  private static final AtomicInteger INVOCATION_COUNT = new AtomicInteger();
+
+  /**
+   * Hooks for testing; save state for ease of asserting on
+   * invocation.
+   */
+  private class Hook implements Runnable {
+
+    private final String name;
+    private final long sleepTime;
+    private final boolean expectFailure;
+    private AssertionError assertion;
+    private boolean invoked;
+    private int invokedOrder;
+    private boolean completed;
+    private boolean interrupted;
+    private long startTime;
+
+    Hook(final String name,
+        final long sleepTime,
+        final boolean expectFailure) {
+      this.name = name;
+      this.sleepTime = sleepTime;
+      this.expectFailure = expectFailure;
+    }
+
+    @Override
+    public void run() {
+      try {
+        invoked = true;
+        invokedOrder = INVOCATION_COUNT.incrementAndGet();
+        startTime = System.currentTimeMillis();
+        LOG.info("Starting shutdown of {} with sleep time of {}",
+            name, sleepTime);
+        if (sleepTime > 0) {
+          sleep(sleepTime);
+        }
+        LOG.info("Completed shutdown of {}", name);
+        completed = true;
+        if (expectFailure) {
+          assertion = new AssertionError("Expected a failure of " + name);
+        }
+      } catch (InterruptedException ex) {
+        LOG.info("Shutdown {} interrupted exception", name, ex);
+        interrupted = true;
+        if (!expectFailure) {
+          assertion = new AssertionError("Timeout of " + name, ex);
+        }
+      }
+      maybeThrowAssertion();
+    }
+
+    /**
+     * Raise any exception generated during the shutdown process.
+     * @throws AssertionError any assertion from the shutdown.
+     */
+    void maybeThrowAssertion() throws AssertionError {
+      if (assertion != null) {
+        throw assertion;
+      }
+    }
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder("Hook{");
+      sb.append("name='").append(name).append('\'');
+      sb.append(", sleepTime=").append(sleepTime);
+      sb.append(", expectFailure=").append(expectFailure);
+      sb.append(", invoked=").append(invoked);
+      sb.append(", invokedOrder=").append(invokedOrder);
+      sb.append(", completed=").append(completed);
+      sb.append(", interrupted=").append(interrupted);
+      sb.append('}');
+      return sb.toString();
+    }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[33/50] [abbrv] hadoop git commit: YARN-9036. Escape newlines in health report in YARN UI. Contributed by Keqiu Hu

Posted by jh...@apache.org.
YARN-9036. Escape newlines in health report in YARN UI. Contributed by Keqiu Hu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9ea6f32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9ea6f32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9ea6f32

Branch: refs/heads/branch-2-jhung-test
Commit: f9ea6f32aeddfbf993d287733f75e79c7009d8fb
Parents: 7144eef
Author: Jonathan Hung <jh...@jhung-ld3.linkedin.biz>
Authored: Fri Nov 30 10:04:06 2018 -0800
Committer: Jonathan Hung <jy...@gmail.com>
Committed: Fri Dec 14 16:03:56 2018 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/server/resourcemanager/webapp/NodesPage.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9ea6f32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index c03df63..c3a4897 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 
 import com.google.inject.Inject;
+import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -200,8 +201,9 @@ class NodesPage extends RmView {
             nodeTableData.length() - 1);
       }
       nodeTableData.append("]");
+      String nodeTableDataEscaped = StringEscapeUtils.escapeJava(nodeTableData.toString());
       html.script().$type("text/javascript")
-          ._("var nodeTableData=" + nodeTableData)._();
+          ._("var nodeTableData=" + nodeTableDataEscaped)._();
       tbody._()._();
     }
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org