You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by gi...@apache.org on 2018/06/29 20:13:52 UTC

[01/50] [abbrv] hadoop git commit: YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan. [Forced Update!]

Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-15461 1c0b63280 -> 75f23cbf6 (forced update)


YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64321286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64321286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64321286

Branch: refs/heads/HADOOP-15461
Commit: 6432128622d64f3f9dd638b9c254c77cdf5408aa
Parents: 30728ac
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 22 17:15:29 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 22 17:15:29 2018 +0000

----------------------------------------------------------------------
 .../org/apache/hadoop/util/SysInfoLinux.java    | 19 ++++++-
 .../apache/hadoop/util/TestSysInfoLinux.java    | 60 ++++++++++++++++++++
 2 files changed, 77 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 7fd1990..2c2aca3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -217,6 +217,21 @@ public class SysInfoLinux extends SysInfo {
   }
 
   /**
+   *
+   * Wrapper for Long.parseLong() that returns zero if the value is
+   * invalid. Under some circumstances, swapFree in /proc/meminfo can
+   * go negative, reported as a very large decimal value.
+   */
+  private long safeParseLong(String strVal) {
+    long parsedVal;
+    try {
+      parsedVal = Long.parseLong(strVal);
+    } catch (NumberFormatException nfe) {
+      parsedVal = 0;
+    }
+    return parsedVal;
+  }
+  /**
    * Read /proc/meminfo, parse and compute memory information.
    * @param readAgain if false, read only on the first time
    */
@@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
           } else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
             swapSize = Long.parseLong(mat.group(2));
           } else if (mat.group(1).equals(MEMFREE_STRING)) {
-            ramSizeFree = Long.parseLong(mat.group(2));
+            ramSizeFree = safeParseLong(mat.group(2));
           } else if (mat.group(1).equals(SWAPFREE_STRING)) {
-            swapSizeFree = Long.parseLong(mat.group(2));
+            swapSizeFree = safeParseLong(mat.group(2));
           } else if (mat.group(1).equals(INACTIVE_STRING)) {
             inactiveSize = Long.parseLong(mat.group(2));
           } else if (mat.group(1).equals(INACTIVEFILE_STRING)) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index a646a41..0ae5d3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -161,6 +161,36 @@ public class TestSysInfoLinux {
     "DirectMap2M:     2027520 kB\n" +
     "DirectMap1G:    132120576 kB\n";
 
+  static final String MEMINFO_FORMAT3 =
+    "MemTotal:      %d kB\n" +
+    "MemFree:         %s kB\n" +
+    "Buffers:        138244 kB\n" +
+    "Cached:         947780 kB\n" +
+    "SwapCached:     142880 kB\n" +
+    "Active:        3229888 kB\n" +
+    "Inactive:       %d kB\n" +
+    "SwapTotal:     %d kB\n" +
+    "SwapFree:      %s kB\n" +
+    "Dirty:          122012 kB\n" +
+    "Writeback:           0 kB\n" +
+    "AnonPages:     2710792 kB\n" +
+    "Mapped:          24740 kB\n" +
+    "Slab:           132528 kB\n" +
+    "SReclaimable:   105096 kB\n" +
+    "SUnreclaim:      27432 kB\n" +
+    "PageTables:      11448 kB\n" +
+    "NFS_Unstable:        0 kB\n" +
+    "Bounce:              0 kB\n" +
+    "CommitLimit:   4125904 kB\n" +
+    "Committed_AS:  4143556 kB\n" +
+    "VmallocTotal: 34359738367 kB\n" +
+    "VmallocUsed:      1632 kB\n" +
+    "VmallocChunk: 34359736375 kB\n" +
+    "HugePages_Total:     %d\n" +
+    "HugePages_Free:      0\n" +
+    "HugePages_Rsvd:      0\n" +
+    "Hugepagesize:     2048 kB";
+
   static final String CPUINFO_FORMAT =
     "processor : %s\n" +
     "vendor_id : AuthenticAMD\n" +
@@ -384,6 +414,36 @@ public class TestSysInfoLinux {
                           (nrHugePages * 2048) + swapTotal));
   }
 
+  /**
+   * Test parsing /proc/meminfo
+   * @throws IOException
+   */
+  @Test
+  public void parsingProcMemFileWithBadValues() throws IOException {
+    long memTotal = 4058864L;
+    long memFree = 0L;  // bad value should return 0
+    long inactive = 567732L;
+    long swapTotal = 2096472L;
+    long swapFree = 0L; // bad value should return 0
+    int nrHugePages = 10;
+    String badFreeValue = "18446744073709551596";
+    File tempFile = new File(FAKE_MEMFILE);
+    tempFile.deleteOnExit();
+    FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+    fWriter.write(String.format(MEMINFO_FORMAT3,
+        memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
+
+    fWriter.close();
+    assertEquals(plugin.getAvailablePhysicalMemorySize(),
+        1024L * (memFree + inactive));
+    assertEquals(plugin.getAvailableVirtualMemorySize(),
+        1024L * (memFree + inactive + swapFree));
+    assertEquals(plugin.getPhysicalMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048)));
+    assertEquals(plugin.getVirtualMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+  }
+
   @Test
   public void testCoreCounts() throws IOException {
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[23/50] [abbrv] hadoop git commit: YARN-8464. Async scheduling thread could be interrupted when there are no NodeManagers in cluster. (Sunil G via wangda)

Posted by gi...@apache.org.
YARN-8464. Async scheduling thread could be interrupted when there are no NodeManagers in cluster. (Sunil G via wangda)

Change-Id: I4f5f856373378685713e77752ba6cf0988a66065


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bedc4fe0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bedc4fe0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bedc4fe0

Branch: refs/heads/HADOOP-15461
Commit: bedc4fe0799cf3b161100acc521fc62a97793427
Parents: ada8f63
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jun 26 19:27:17 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jun 26 19:27:17 2018 -0700

----------------------------------------------------------------------
 .../scheduler/capacity/CapacityScheduler.java     | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bedc4fe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 50ab70d..54bbf24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -521,7 +521,14 @@ public class CapacityScheduler extends
     // First randomize the start point
     int current = 0;
     Collection<FiCaSchedulerNode> nodes = cs.nodeTracker.getAllNodes();
-    int start = random.nextInt(nodes.size());
+
+    // If nodes size is 0 (when there are no node managers registered,
+    // we can return from here itself.
+    int nodeSize = nodes.size();
+    if(nodeSize == 0) {
+      return;
+    }
+    int start = random.nextInt(nodeSize);
 
     // To avoid too verbose DEBUG logging, only print debug log once for
     // every 10 secs.
@@ -574,6 +581,7 @@ public class CapacityScheduler extends
 
     @Override
     public void run() {
+      int debuggingLogCounter = 0;
       while (!Thread.currentThread().isInterrupted()) {
         try {
           if (!runSchedules.get()) {
@@ -585,6 +593,14 @@ public class CapacityScheduler extends
               Thread.sleep(1);
             } else{
               schedule(cs);
+              if(LOG.isDebugEnabled()) {
+                // Adding a debug log here to ensure that the thread is alive
+                // and running fine.
+                if (debuggingLogCounter++ > 10000) {
+                  debuggingLogCounter = 0;
+                  LOG.debug("AsyncScheduleThread[" + getName() + "] is running!");
+                }
+              }
             }
           }
         } catch (InterruptedException ie) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[45/50] [abbrv] hadoop git commit: YARN-8469. [UI2] URL needs to be trimmed to handle index.html redirection while accessing via knox. Contributed by Sunil Govindan.

Posted by gi...@apache.org.
YARN-8469. [UI2] URL needs to be trimmed to handle index.html redirection while accessing via knox. Contributed by Sunil Govindan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a820738e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a820738e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a820738e

Branch: refs/heads/HADOOP-15461
Commit: a820738e347854aed7cdaa7758d9c0830df624e3
Parents: 469b29c
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Fri Jun 29 10:21:25 2018 -0700
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Fri Jun 29 10:21:25 2018 -0700

----------------------------------------------------------------------
 .../src/main/webapp/app/initializers/loader.js            | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a820738e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
index 6642869..96c8fcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
@@ -205,5 +205,13 @@ export default {
 
 const skipTrailingSlash = function(path) {
   path = path.replace('ui2/', '');
-  return path.replace(/\/$/, '');
+  path = path.replace(/\/$/, '');
+  console.log('base url:' + path)
+  if(path.includes("redirect")) {
+    var to = path.lastIndexOf('/');
+    to = to == -1 ? path.length : to + 1;
+    path = path.substring(0, to);
+    console.log('base url after redirect:' + path)
+  }
+  return path;
 };


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[49/50] [abbrv] hadoop git commit: HADOOP-15465. Deprecate WinUtils#Symlinks by using native java code. Contributed by Giovanni Matteo Fumarola.

Posted by gi...@apache.org.
HADOOP-15465. Deprecate WinUtils#Symlinks by using native java code. Contributed by Giovanni Matteo Fumarola.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/32372eac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/32372eac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/32372eac

Branch: refs/heads/HADOOP-15461
Commit: 32372eacbfc13451e4be6254602b38f632582b0c
Parents: cdb0844
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 7 17:02:01 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 13:08:57 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/FileUtil.java     | 60 ++++++++------------
 .../apache/hadoop/fs/RawLocalFileSystem.java    |  2 -
 .../main/java/org/apache/hadoop/util/Shell.java |  9 ++-
 .../hadoop/yarn/server/MiniYARNCluster.java     | 13 ++---
 4 files changed, 37 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index df89598..61cb8d2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -34,8 +34,10 @@ import java.net.URI;
 import java.net.UnknownHostException;
 import java.nio.charset.Charset;
 import java.nio.file.AccessDeniedException;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
@@ -1028,17 +1030,15 @@ public class FileUtil {
   }
 
   /**
-   * Create a soft link between a src and destination
-   * only on a local disk. HDFS does not support this.
-   * On Windows, when symlink creation fails due to security
-   * setting, we will log a warning. The return code in this
-   * case is 2.
+   * Create a soft link between a src and destination only on a local disk. On
+   * Windows, when symlink creation fails due to security setting, we will log a
+   * warning. The return code in this case is 2.
    *
    * @param target the target for symlink
    * @param linkname the symlink
    * @return 0 on success
    */
-  public static int symLink(String target, String linkname) throws IOException{
+  public static int symLink(String target, String linkname) throws IOException {
 
     if (target == null || linkname == null) {
       LOG.warn("Can not create a symLink with a target = " + target
@@ -1053,44 +1053,32 @@ public class FileUtil {
     File linkFile = new File(
         Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
 
-    String[] cmd = Shell.getSymlinkCommand(
-        targetFile.toString(),
-        linkFile.toString());
-
-    ShellCommandExecutor shExec;
     try {
-      if (Shell.WINDOWS &&
-          linkFile.getParentFile() != null &&
-          !new Path(target).isAbsolute()) {
-        // Relative links on Windows must be resolvable at the time of
-        // creation. To ensure this we run the shell command in the directory
-        // of the link.
-        //
-        shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
-      } else {
-        shExec = new ShellCommandExecutor(cmd);
-      }
-      shExec.execute();
-    } catch (Shell.ExitCodeException ec) {
-      int returnVal = ec.getExitCode();
-      if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
-        LOG.warn("Fail to create symbolic links on Windows. "
-            + "The default security settings in Windows disallow non-elevated "
-            + "administrators and all non-administrators from creating symbolic links. "
-            + "This behavior can be changed in the Local Security Policy management console");
-      } else if (returnVal != 0) {
-        LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
-            + returnVal + " with: " + ec.getMessage());
-      }
-      return returnVal;
+      Files.createSymbolicLink(Paths.get(linkFile.toString()),
+          Paths.get(targetFile.toString()));
+    } catch (SecurityException e3) {
+      LOG.warn("Fail to create symbolic links on Windows. "
+          + "The default security settings in Windows disallow non-elevated "
+          + "administrators and all non-administrators from creating symbolic"
+          + " links. This behavior can be changed in the Local Security Policy"
+          + " management console");
+      return SYMLINK_NO_PRIVILEGE;
+
+    } catch (FileAlreadyExistsException | UnsupportedOperationException e) {
+      LOG.warn("Fail to create symbolic links. ErrorMessage = "
+          + e.getLocalizedMessage());
+      return 1;
+
     } catch (IOException e) {
       if (LOG.isDebugEnabled()) {
         LOG.debug("Error while create symlink " + linkname + " to " + target
             + "." + " Exception: " + StringUtils.stringifyException(e));
       }
       throw e;
+
     }
-    return shExec.getExitCode();
+
+    return 0;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index bd003ae..8333366 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -925,7 +925,6 @@ public class RawLocalFileSystem extends FileSystem {
     return true;
   }
 
-  @SuppressWarnings("deprecation")
   @Override
   public void createSymlink(Path target, Path link, boolean createParent)
       throws IOException {
@@ -941,7 +940,6 @@ public class RawLocalFileSystem extends FileSystem {
       mkdirs(link.getParent());
     }
 
-    // NB: Use createSymbolicLink in java.nio.file.Path once available
     int result = FileUtil.symLink(target.toString(),
         makeAbsolute(link).toString());
     if (result != 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 46a0fcc..e902af0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -37,7 +37,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -297,7 +296,13 @@ public abstract class Shell {
         : new String[] { "chown", owner };
   }
 
-  /** Return a command to create symbolic links. */
+  /**
+   * Return a command to create symbolic links.
+   *
+   * Deprecated and likely to be deleted in the near future. Please use
+   * FileUtil.symlink().
+   */
+  @Deprecated
   public static String[] getSymlinkCommand(String target, String link) {
     return WINDOWS ?
        new String[] { getWinUtilsPath(), "symlink", link, target }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 0395138..01ac02c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.ha.HAServiceProtocol;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -42,7 +43,6 @@ import org.apache.hadoop.net.ServerSocketUtil;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.conf.HAUtil;
@@ -202,14 +202,13 @@ public class MiniYARNCluster extends CompositeService {
       // Guarantee target exists before creating symlink.
       targetWorkDir.mkdirs();
 
-      ShellCommandExecutor shexec = new ShellCommandExecutor(
-        Shell.getSymlinkCommand(targetPath, linkPath));
       try {
-        shexec.execute();
+        FileUtil.symLink(targetPath, linkPath);
       } catch (IOException e) {
-        throw new YarnRuntimeException(String.format(
-          "failed to create symlink from %s to %s, shell output: %s", linkPath,
-          targetPath, shexec.getOutput()), e);
+        throw new YarnRuntimeException(
+            String.format("failed to create symlink from %s to %s.",
+                linkPath, targetPath),
+            e);
       }
 
       this.testWorkDir = link;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[26/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton

Posted by gi...@apache.org.
HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaf03cc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaf03cc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaf03cc4

Branch: refs/heads/HADOOP-15461
Commit: aaf03cc459a34af284f9735453aefd4ddb430d67
Parents: fbaff36
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 12:39:15 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 12:39:15 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  26 --
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   1 -
 .../common/src/main/resources/ozone-default.xml |  47 ---
 .../container/replication/ReplicationQueue.java |  78 +++++
 .../replication/ReplicationReqMsg.java          | 107 ++++++
 .../container/replication/package-info.java     |  23 ++
 .../replication/TestReplicationQueue.java       | 134 ++++++++
 .../container/replication/package-info.java     |  23 ++
 .../hdds/scm/container/ContainerMapping.java    |  10 +-
 .../replication/ContainerSupervisor.java        | 340 -------------------
 .../container/replication/InProgressPool.java   | 255 --------------
 .../scm/container/replication/PeriodicPool.java | 119 -------
 .../scm/container/replication/package-info.java |  23 --
 .../hadoop/hdds/scm/node/NodeManager.java       |   6 -
 .../hadoop/hdds/scm/node/NodePoolManager.java   |  71 ----
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  23 --
 .../hdds/scm/node/SCMNodePoolManager.java       | 269 ---------------
 .../hdds/scm/container/MockNodeManager.java     |   6 -
 .../hdds/scm/node/TestSCMNodePoolManager.java   | 160 ---------
 .../testutils/ReplicationNodeManagerMock.java   |   5 -
 .../ReplicationNodePoolManagerMock.java         | 133 --------
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  31 --
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  74 ----
 23 files changed, 368 insertions(+), 1596 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 85407e6..df6fbf0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,32 +243,6 @@ public final class ScmConfigKeys {
   public static final String
       OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
 
-  /**
-   * Don't start processing a pool if we have not had a minimum number of
-   * seconds from the last processing.
-   */
-  public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
-      "ozone.scm.container.report.processing.interval";
-  public static final String
-      OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
-  /**
-   * This determines the total number of pools to be processed in parallel.
-   */
-  public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
-      "ozone.scm.max.nodepool.processing.threads";
-  public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
-  /**
-   * These 2 settings control the number of threads in executor pool and time
-   * outs for thw container reports from all nodes.
-   */
-  public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
-      "ozone.scm.max.container.report.threads";
-  public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
-      "ozone.scm.container.reports.wait.timeout";
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
-      "5m";
 
   public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
       "ozone.scm.block.deletion.max.retry";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index c40dc8e..08a5ffd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,7 +91,6 @@ public final class OzoneConsts {
   public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
   public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
   public static final String BLOCK_DB = "block.db";
-  public static final String NODEPOOL_DB = "nodepool.db";
   public static final String OPEN_CONTAINERS_DB = "openContainers.db";
   public static final String DELETED_BLOCK_DB = "deletedBlock.db";
   public static final String KSM_DB_NAME = "ksm.db";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7a91610..25365c8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,25 +572,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.container.report.processing.interval</name>
-    <value>60s</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>Time interval for scm to process container reports
-      for a node pool. Scm handles node pool reports in a cyclic clock
-      manner, it fetches pools periodically with this time interval.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.reports.wait.timeout</name>
-    <value>300s</value>
-    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
-    <description>Maximum time to wait in seconds for processing all container
-      reports from
-      a node pool. It determines the timeout for a
-      node pool report.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.container.size.gb</name>
     <value>5</value>
     <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -793,17 +774,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.container.report.threads</name>
-    <value>100</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Maximum number of threads to process container reports in scm.
-      Each container report from a data node is processed by scm in a worker
-      thread, fetched from a thread pool. This property is used to control the
-      maximum size of the thread pool.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.max.hb.count.to.process</name>
     <value>5000</value>
     <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -815,14 +785,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      Number of node pools to process in parallel.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.names</name>
     <value/>
     <tag>OZONE</tag>
@@ -844,15 +806,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      Controls the number of node pools that can be processed in parallel by
-      Container Supervisor.
-    </description>
-  </property>
-  <property>
     <name>ozone.trace.enabled</name>
     <value>false</value>
     <tag>OZONE, DEBUG</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
new file mode 100644
index 0000000..b83ecf1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+
+/**
+ * Priority queue to handle under-replicated and over replicated containers
+ * in ozone. ReplicationManager will consume these messages and decide
+ * accordingly.
+ */
+public class ReplicationQueue {
+
+  private final Queue<ReplicationReqMsg> queue;
+
+  ReplicationQueue() {
+    queue = new PriorityQueue<>();
+  }
+
+  public synchronized boolean add(ReplicationReqMsg repObj) {
+    if (this.queue.contains(repObj)) {
+      // Remove the earlier message and insert this one
+      this.queue.remove(repObj);
+      return this.queue.add(repObj);
+    } else {
+      return this.queue.add(repObj);
+    }
+  }
+
+  public synchronized boolean remove(ReplicationReqMsg repObj) {
+    return queue.remove(repObj);
+  }
+
+  /**
+   * Retrieves, but does not remove, the head of this queue,
+   * or returns {@code null} if this queue is empty.
+   *
+   * @return the head of this queue, or {@code null} if this queue is empty
+   */
+  public synchronized ReplicationReqMsg peek() {
+    return queue.peek();
+  }
+
+  /**
+   * Retrieves and removes the head of this queue,
+   * or returns {@code null} if this queue is empty.
+   *
+   * @return the head of this queue, or {@code null} if this queue is empty
+   */
+  public synchronized ReplicationReqMsg poll() {
+    return queue.poll();
+  }
+
+  public synchronized boolean removeAll(List<ReplicationReqMsg> repObjs) {
+    return queue.removeAll(repObjs);
+  }
+
+  public int size() {
+    return queue.size();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
new file mode 100644
index 0000000..8d26fc3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.math.NumberUtils;
+
+/**
+ * Wrapper class for hdds replication queue. Implements its natural
+ * ordering for priority queue.
+ */
+public class ReplicationReqMsg implements Comparable<ReplicationReqMsg>,
+    Serializable {
+  private final long containerId;
+  private final short replicationCount;
+  private final short expecReplicationCount;
+  private final long timestamp;
+
+  public ReplicationReqMsg(long containerId, short replicationCount,
+      long timestamp, short expecReplicationCount) {
+    this.containerId = containerId;
+    this.replicationCount = replicationCount;
+    this.timestamp = timestamp;
+    this.expecReplicationCount = expecReplicationCount;
+  }
+
+  /**
+   * Compares this object with the specified object for order.  Returns a
+   * negative integer, zero, or a positive integer as this object is less
+   * than, equal to, or greater than the specified object.
+   * @param o the object to be compared.
+   * @return a negative integer, zero, or a positive integer as this object
+   * is less than, equal to, or greater than the specified object.
+   * @throws NullPointerException if the specified object is null
+   * @throws ClassCastException   if the specified object's type prevents it
+   *                              from being compared to this object.
+   */
+  @Override
+  public int compareTo(ReplicationReqMsg o) {
+    if (this == o) {
+      return 0;
+    }
+    if (o == null) {
+      return 1;
+    }
+    int retVal = NumberUtils
+        .compare(getReplicationCount() - getExpecReplicationCount(),
+            o.getReplicationCount() - o.getExpecReplicationCount());
+    if (retVal != 0) {
+      return retVal;
+    }
+    return NumberUtils.compare(getTimestamp(), o.getTimestamp());
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(91, 1011)
+        .append(getContainerId())
+        .toHashCode();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ReplicationReqMsg that = (ReplicationReqMsg) o;
+    return new EqualsBuilder().append(getContainerId(), that.getContainerId())
+        .isEquals();
+  }
+
+  public long getContainerId() {
+    return containerId;
+  }
+
+  public short getReplicationCount() {
+    return replicationCount;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public short getExpecReplicationCount() {
+    return expecReplicationCount;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..7f335e3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Ozone Container replicaton related classes.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
new file mode 100644
index 0000000..39c61d3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.Random;
+import java.util.UUID;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for ReplicationQueue.
+ */
+public class TestReplicationQueue {
+
+  private ReplicationQueue replicationQueue;
+  private Random random;
+
+  @Before
+  public void setUp() {
+    replicationQueue = new ReplicationQueue();
+    random = new Random();
+  }
+
+  @Test
+  public void testDuplicateAddOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationReqMsg obj1, obj2, obj3;
+    long time = Time.monotonicNow();
+    obj1 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
+    obj2 = new ReplicationReqMsg(contId, (short) 2, time + 1, (short) 3);
+    obj3 = new ReplicationReqMsg(contId, (short) 1, time+2, (short) 3);
+
+    replicationQueue.add(obj1);
+    replicationQueue.add(obj2);
+    replicationQueue.add(obj3);
+    Assert.assertEquals("Should add only 1 msg as second one is duplicate",
+        1, replicationQueue.size());
+    ReplicationReqMsg temp = replicationQueue.poll();
+    Assert.assertEquals(temp, obj3);
+  }
+
+  @Test
+  public void testPollOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationReqMsg msg1, msg2, msg3, msg4, msg5;
+    msg1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
+        (short) 3);
+    long time = Time.monotonicNow();
+    msg2 = new ReplicationReqMsg(contId + 1, (short) 4, time, (short) 3);
+    msg3 = new ReplicationReqMsg(contId + 2, (short) 0, time, (short) 3);
+    msg4 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
+    // Replication message for same container but different nodeId
+    msg5 = new ReplicationReqMsg(contId + 1, (short) 2, time, (short) 3);
+
+    replicationQueue.add(msg1);
+    replicationQueue.add(msg2);
+    replicationQueue.add(msg3);
+    replicationQueue.add(msg4);
+    replicationQueue.add(msg5);
+    Assert.assertEquals("Should have 3 objects",
+        3, replicationQueue.size());
+
+    // Since Priority queue orders messages according to replication count,
+    // message with lowest replication should be first
+    ReplicationReqMsg temp;
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 2 objects",
+        2, replicationQueue.size());
+    Assert.assertEquals(temp, msg3);
+
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 1 objects",
+        1, replicationQueue.size());
+    Assert.assertEquals(temp, msg5);
+
+    // Message 2 should be ordered before message 5 as both have same replication
+    // number but message 2 has earlier timestamp.
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 0 objects",
+        replicationQueue.size(), 0);
+    Assert.assertEquals(temp, msg4);
+  }
+
+  @Test
+  public void testRemoveOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationReqMsg obj1, obj2, obj3;
+    obj1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
+        (short) 3);
+    obj2 = new ReplicationReqMsg(contId + 1, (short) 2, Time.monotonicNow(),
+        (short) 3);
+    obj3 = new ReplicationReqMsg(contId + 2, (short) 3, Time.monotonicNow(),
+        (short) 3);
+
+    replicationQueue.add(obj1);
+    replicationQueue.add(obj2);
+    replicationQueue.add(obj3);
+    Assert.assertEquals("Should have 3 objects",
+        3, replicationQueue.size());
+
+    replicationQueue.remove(obj3);
+    Assert.assertEquals("Should have 2 objects",
+        2, replicationQueue.size());
+
+    replicationQueue.remove(obj2);
+    Assert.assertEquals("Should have 1 objects",
+        1, replicationQueue.size());
+
+    replicationQueue.remove(obj1);
+    Assert.assertEquals("Should have 0 objects",
+        0, replicationQueue.size());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5b1fd0f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * SCM Testing and Mocking Utils.
+ */
+package org.apache.hadoop.ozone.container.replication;
+// Test classes for Replication functionality.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index b563e90..9fd30f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -80,7 +79,6 @@ public class ContainerMapping implements Mapping {
   private final PipelineSelector pipelineSelector;
   private final ContainerStateManager containerStateManager;
   private final LeaseManager<ContainerInfo> containerLeaseManager;
-  private final ContainerSupervisor containerSupervisor;
   private final float containerCloseThreshold;
   private final ContainerCloser closer;
   private final long size;
@@ -127,9 +125,7 @@ public class ContainerMapping implements Mapping {
         OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
     this.containerStateManager =
         new ContainerStateManager(conf, this);
-    this.containerSupervisor =
-        new ContainerSupervisor(conf, nodeManager,
-            nodeManager.getNodePoolManager());
+
     this.containerCloseThreshold = conf.getFloat(
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -407,8 +403,8 @@ public class ContainerMapping implements Mapping {
       throws IOException {
     List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
         containerInfos = reports.getReportsList();
-    containerSupervisor.handleContainerReport(datanodeDetails, reports);
-    for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+
+     for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
         containerInfos) {
       byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
       lock.lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index 5bd0574..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
-    .sleepUninterruptibly;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(ContainerSupervisor.class);
-
-  private final NodePoolManager poolManager;
-  private final HashSet<String> poolNames;
-  private final PriorityQueue<PeriodicPool> poolQueue;
-  private final NodeManager nodeManager;
-  private final long containerProcessingLag;
-  private final AtomicBoolean runnable;
-  private final ExecutorService executorService;
-  private final long maxPoolWait;
-  private long poolProcessCount;
-  private final List<InProgressPool> inProgressPoolList;
-  private final AtomicInteger threadFaultCount;
-  private final int inProgressPoolMaxCount;
-
-  private final ReadWriteLock inProgressPoolListLock;
-
-  /**
-   * Returns the number of times we have processed pools.
-   * @return long
-   */
-  public long getPoolProcessCount() {
-    return poolProcessCount;
-  }
-
-
-  /**
-   * Constructs a class that computes Replication Levels.
-   *
-   * @param conf - OzoneConfiguration
-   * @param nodeManager - Node Manager
-   * @param poolManager - Pool Manager
-   */
-  public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
-                             NodePoolManager poolManager) {
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(nodeManager);
-    this.containerProcessingLag =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
-            OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
-            TimeUnit.SECONDS
-        ) * 1000;
-    int maxContainerReportThreads =
-        conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
-            OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
-        );
-    this.maxPoolWait =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
-            OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
-            TimeUnit.MILLISECONDS);
-    this.inProgressPoolMaxCount = conf.getInt(
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
-    this.poolManager = poolManager;
-    this.nodeManager = nodeManager;
-    this.poolNames = new HashSet<>();
-    this.poolQueue = new PriorityQueue<>();
-    this.runnable = new AtomicBoolean(true);
-    this.threadFaultCount = new AtomicInteger(0);
-    this.executorService = newCachedThreadPool(
-        new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("Container Reports Processing Thread - %d")
-            .build(), maxContainerReportThreads);
-    this.inProgressPoolList = new LinkedList<>();
-    this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
-    initPoolProcessThread();
-  }
-
-  private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
-      int maxThreads) {
-    return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
-        new LinkedBlockingQueue<>(), threadFactory);
-  }
-
-  /**
-   * Returns the number of pools that are under process right now.
-   * @return  int - Number of pools that are in process.
-   */
-  public int getInProgressPoolCount() {
-    return inProgressPoolList.size();
-  }
-
-  /**
-   * Exits the background thread.
-   */
-  public void setExit() {
-    this.runnable.set(false);
-  }
-
-  /**
-   * Adds or removes pools from names that we need to process.
-   *
-   * There are two different cases that we need to process.
-   * The case where some pools are being added and some times we have to
-   * handle cases where pools are removed.
-   */
-  private void refreshPools() {
-    List<String> pools = this.poolManager.getNodePools();
-    if (pools != null) {
-
-      HashSet<String> removedPools =
-          computePoolDifference(this.poolNames, new HashSet<>(pools));
-
-      HashSet<String> addedPools =
-          computePoolDifference(new HashSet<>(pools), this.poolNames);
-      // TODO: Support remove pool API in pool manager so that this code
-      // path can be tested. This never happens in the current code base.
-      for (String poolName : removedPools) {
-        for (PeriodicPool periodicPool : poolQueue) {
-          if (periodicPool.getPoolName().compareTo(poolName) == 0) {
-            poolQueue.remove(periodicPool);
-          }
-        }
-      }
-      // Remove the pool names that we have in the list.
-      this.poolNames.removeAll(removedPools);
-
-      for (String poolName : addedPools) {
-        poolQueue.add(new PeriodicPool(poolName));
-      }
-
-      // Add to the pool names we are tracking.
-      poolNames.addAll(addedPools);
-    }
-
-  }
-
-  /**
-   * Handle the case where pools are added.
-   *
-   * @param newPools - New Pools list
-   * @param oldPool - oldPool List.
-   */
-  private HashSet<String> computePoolDifference(HashSet<String> newPools,
-      Set<String> oldPool) {
-    Preconditions.checkNotNull(newPools);
-    Preconditions.checkNotNull(oldPool);
-    HashSet<String> newSet = new HashSet<>(newPools);
-    newSet.removeAll(oldPool);
-    return newSet;
-  }
-
-  private void initPoolProcessThread() {
-
-    /*
-     * Task that runs to check if we need to start a pool processing job.
-     * if so we create a pool reconciliation job and find out of all the
-     * expected containers are on the nodes.
-     */
-    Runnable processPools = () -> {
-      while (runnable.get()) {
-        // Make sure that we don't have any new pools.
-        refreshPools();
-        while (inProgressPoolList.size() < inProgressPoolMaxCount) {
-          PeriodicPool pool = poolQueue.poll();
-          if (pool != null) {
-            if (pool.getLastProcessedTime() + this.containerProcessingLag >
-                Time.monotonicNow()) {
-              LOG.debug("Not within the time window for processing: {}",
-                  pool.getPoolName());
-              // we might over sleep here, not a big deal.
-              sleepUninterruptibly(this.containerProcessingLag,
-                  TimeUnit.MILLISECONDS);
-            }
-            LOG.debug("Adding pool {} to container processing queue",
-                pool.getPoolName());
-            InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
-                pool, this.nodeManager, this.poolManager, this.executorService);
-            inProgressPool.startReconciliation();
-            inProgressPoolListLock.writeLock().lock();
-            try {
-              inProgressPoolList.add(inProgressPool);
-            } finally {
-              inProgressPoolListLock.writeLock().unlock();
-            }
-            poolProcessCount++;
-          } else {
-            break;
-          }
-        }
-        sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
-        inProgressPoolListLock.readLock().lock();
-        try {
-          for (InProgressPool inProgressPool : inProgressPoolList) {
-            inProgressPool.finalizeReconciliation();
-            poolQueue.add(inProgressPool.getPool());
-          }
-        } finally {
-          inProgressPoolListLock.readLock().unlock();
-        }
-        inProgressPoolListLock.writeLock().lock();
-        try {
-          inProgressPoolList.clear();
-        } finally {
-          inProgressPoolListLock.writeLock().unlock();
-        }
-      }
-    };
-
-    // We will have only one thread for pool processing.
-    Thread poolProcessThread = new Thread(processPools);
-    poolProcessThread.setDaemon(true);
-    poolProcessThread.setName("Pool replica thread");
-    poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
-      // Let us just restart this thread after logging a critical error.
-      // if this thread is not running we cannot handle commands from SCM.
-      LOG.error("Critical Error : Pool replica thread encountered an " +
-          "error. Thread: {} Error Count : {}", t.toString(), e,
-          threadFaultCount.incrementAndGet());
-      poolProcessThread.start();
-      // TODO : Add a config to restrict how many times we will restart this
-      // thread in a single session.
-    });
-    poolProcessThread.start();
-  }
-
-  /**
-   * Adds a container report to appropriate inProgress Pool.
-   * @param containerReport  -- Container report for a specific container from
-   * a datanode.
-   */
-  public void handleContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto containerReport) {
-    inProgressPoolListLock.readLock().lock();
-    try {
-      String poolName = poolManager.getNodePool(datanodeDetails);
-      for (InProgressPool ppool : inProgressPoolList) {
-        if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
-          ppool.handleContainerReport(datanodeDetails, containerReport);
-          return;
-        }
-      }
-      // TODO: Decide if we can do anything else with this report.
-      LOG.debug("Discarding the container report for pool {}. " +
-              "That pool is not currently in the pool reconciliation process." +
-              " Container Name: {}", poolName, datanodeDetails);
-    } catch (SCMException e) {
-      LOG.warn("Skipping processing container report from datanode {}, "
-              + "cause: failed to get the corresponding node pool",
-          datanodeDetails.toString(), e);
-    } finally {
-      inProgressPoolListLock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get in process pool list, used for testing.
-   * @return List of InProgressPool
-   */
-  @VisibleForTesting
-  public List<InProgressPool> getInProcessPoolList() {
-    return inProgressPoolList;
-  }
-
-  /**
-   * Shutdown the Container Replication Manager.
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    setExit();
-    HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 4b54731..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(InProgressPool.class);
-
-  private final PeriodicPool pool;
-  private final NodeManager nodeManager;
-  private final NodePoolManager poolManager;
-  private final ExecutorService executorService;
-  private final Map<Long, Integer> containerCountMap;
-  private final Map<UUID, Boolean> processedNodeSet;
-  private final long startTime;
-  private ProgressStatus status;
-  private AtomicInteger nodeCount;
-  private AtomicInteger nodeProcessed;
-  private AtomicInteger containerProcessedCount;
-  private long maxWaitTime;
-  /**
-   * Constructs an pool that is being processed.
-   *  @param maxWaitTime - Maximum wait time in milliseconds.
-   * @param pool - Pool that we are working against
-   * @param nodeManager - Nodemanager
-   * @param poolManager - pool manager
-   * @param executorService - Shared Executor service.
-   */
-  InProgressPool(long maxWaitTime, PeriodicPool pool,
-      NodeManager nodeManager, NodePoolManager poolManager,
-                 ExecutorService executorService) {
-    Preconditions.checkNotNull(pool);
-    Preconditions.checkNotNull(nodeManager);
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(executorService);
-    Preconditions.checkArgument(maxWaitTime > 0);
-    this.pool = pool;
-    this.nodeManager = nodeManager;
-    this.poolManager = poolManager;
-    this.executorService = executorService;
-    this.containerCountMap = new ConcurrentHashMap<>();
-    this.processedNodeSet = new ConcurrentHashMap<>();
-    this.maxWaitTime = maxWaitTime;
-    startTime = Time.monotonicNow();
-  }
-
-  /**
-   * Returns periodic pool.
-   *
-   * @return PeriodicPool
-   */
-  public PeriodicPool getPool() {
-    return pool;
-  }
-
-  /**
-   * We are done if we have got reports from all nodes or we have
-   * done waiting for the specified time.
-   *
-   * @return true if we are done, false otherwise.
-   */
-  public boolean isDone() {
-    return (nodeCount.get() == nodeProcessed.get()) ||
-        (this.startTime + this.maxWaitTime) > Time.monotonicNow();
-  }
-
-  /**
-   * Gets the number of containers processed.
-   *
-   * @return int
-   */
-  public int getContainerProcessedCount() {
-    return containerProcessedCount.get();
-  }
-
-  /**
-   * Returns the start time in milliseconds.
-   *
-   * @return - Start Time.
-   */
-  public long getStartTime() {
-    return startTime;
-  }
-
-  /**
-   * Get the number of nodes in this pool.
-   *
-   * @return - node count
-   */
-  public int getNodeCount() {
-    return nodeCount.get();
-  }
-
-  /**
-   * Get the number of nodes that we have already processed container reports
-   * from.
-   *
-   * @return - Processed count.
-   */
-  public int getNodeProcessed() {
-    return nodeProcessed.get();
-  }
-
-  /**
-   * Returns the current status.
-   *
-   * @return Status
-   */
-  public ProgressStatus getStatus() {
-    return status;
-  }
-
-  /**
-   * Starts the reconciliation process for all the nodes in the pool.
-   */
-  public void startReconciliation() {
-    List<DatanodeDetails> datanodeDetailsList =
-        this.poolManager.getNodes(pool.getPoolName());
-    if (datanodeDetailsList.size() == 0) {
-      LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
-          pool.getPoolName());
-      this.status = ProgressStatus.Error;
-      return;
-    }
-
-    nodeProcessed = new AtomicInteger(0);
-    containerProcessedCount = new AtomicInteger(0);
-    nodeCount = new AtomicInteger(0);
-    this.status = ProgressStatus.InProgress;
-    this.getPool().setLastProcessedTime(Time.monotonicNow());
-  }
-
-  /**
-   * Queues a container Report for handling. This is done in a worker thread
-   * since decoding a container report might be compute intensive . We don't
-   * want to block since we have asked for bunch of container reports
-   * from a set of datanodes.
-   *
-   * @param containerReport - ContainerReport
-   */
-  public void handleContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto containerReport) {
-    if (status == ProgressStatus.InProgress) {
-      executorService.submit(processContainerReport(datanodeDetails,
-          containerReport));
-    } else {
-      LOG.debug("Cannot handle container report when the pool is in {} status.",
-          status);
-    }
-  }
-
-  private Runnable processContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto reports) {
-    return () -> {
-      if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
-          (k) -> true)) {
-        nodeProcessed.incrementAndGet();
-        LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
-            datanodeDetails.getUuid());
-        for (ContainerInfo info : reports.getReportsList()) {
-          containerProcessedCount.incrementAndGet();
-          LOG.debug("Total Containers processed: {} Container Name: {}",
-              containerProcessedCount.get(), info.getContainerID());
-
-          // Update the container map with count + 1 if the key exists or
-          // update the map with 1. Since this is a concurrentMap the
-          // computation and update is atomic.
-          containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
-        }
-      }
-    };
-  }
-
-  /**
-   * Filter the containers based on specific rules.
-   *
-   * @param predicate -- Predicate to filter by
-   * @return A list of map entries.
-   */
-  public List<Map.Entry<Long, Integer>> filterContainer(
-      Predicate<Map.Entry<Long, Integer>> predicate) {
-    return containerCountMap.entrySet().stream()
-        .filter(predicate).collect(Collectors.toList());
-  }
-
-  /**
-   * Used only for testing, calling this will abort container report
-   * processing. This is very dangerous call and should not be made by any users
-   */
-  @VisibleForTesting
-  public void setDoneProcessing() {
-    nodeProcessed.set(nodeCount.get());
-  }
-
-  /**
-   * Returns the pool name.
-   *
-   * @return Name of the pool.
-   */
-  String getPoolName() {
-    return pool.getPoolName();
-  }
-
-  public void finalizeReconciliation() {
-    status = ProgressStatus.Done;
-    //TODO: Add finalizing logic. This is where actual reconciliation happens.
-  }
-
-  /**
-   * Current status of the computing replication status.
-   */
-  public enum ProgressStatus {
-    InProgress, Done, Error
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index ef28aa7..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
-  private final String poolName;
-  private long lastProcessedTime;
-  private AtomicLong totalProcessedCount;
-
-  /**
-   * Constructs a periodic pool.
-   *
-   * @param poolName - Name of the pool
-   */
-  public PeriodicPool(String poolName) {
-    this.poolName = poolName;
-    lastProcessedTime = 0;
-    totalProcessedCount = new AtomicLong(0);
-  }
-
-  /**
-   * Get pool Name.
-   * @return PoolName
-   */
-  public String getPoolName() {
-    return poolName;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(PeriodicPool o) {
-    return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    PeriodicPool that = (PeriodicPool) o;
-
-    return poolName.equals(that.poolName);
-  }
-
-  @Override
-  public int hashCode() {
-    return poolName.hashCode();
-  }
-
-  /**
-   * Returns the Total Times we have processed this pool.
-   *
-   * @return processed count.
-   */
-  public long getTotalProcessedCount() {
-    return totalProcessedCount.get();
-  }
-
-  /**
-   * Gets the last time we processed this pool.
-   * @return time in milliseconds
-   */
-  public long getLastProcessedTime() {
-    return this.lastProcessedTime;
-  }
-
-
-  /**
-   * Sets the last processed time.
-   *
-   * @param lastProcessedTime - Long in milliseconds.
-   */
-
-  public void setLastProcessedTime(long lastProcessedTime) {
-    this.lastProcessedTime = lastProcessedTime;
-  }
-
-  /*
-   * Increments the total processed count.
-   */
-  public void incTotalProcessedCount() {
-    this.totalProcessedCount.incrementAndGet();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
deleted file mode 100644
index 7bbe2ef..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 4392633..72d7e94 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,12 +124,6 @@ public interface NodeManager extends StorageContainerNodeProtocol,
   SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
 
   /**
-   * Returns the NodePoolManager associated with the NodeManager.
-   * @return NodePoolManager
-   */
-  NodePoolManager getNodePoolManager();
-
-  /**
    * Wait for the heartbeat is processed by NodeManager.
    * @return true if heartbeat has been processed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
deleted file mode 100644
index 46faf9ca..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
-  /**
-   * Add a node to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   */
-  void addNode(String pool, DatanodeDetails node) throws IOException;
-
-  /**
-   * Remove a node from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   * @throws SCMException
-   */
-  void removeNode(String pool, DatanodeDetails node)
-      throws SCMException;
-
-  /**
-   * Get a list of known node pools.
-   * @return a list of known node pool names or an empty list if not node pool
-   * is defined.
-   */
-  List<String> getNodePools();
-
-  /**
-   * Get all nodes of a node pool given the name of the node pool.
-   * @param pool - name of the node pool.
-   * @return a list of datanode ids or an empty list if the node pool was not
-   *  found.
-   */
-  List<DatanodeDetails> getNodes(String pool);
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeDetails - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   */
-  String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fc8b013..adca8ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
-import com.google.protobuf.GeneratedMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -159,7 +158,6 @@ public class SCMNodeManager
   private ObjectName nmInfoBean;
 
   // Node pool manager.
-  private final SCMNodePoolManager nodePoolManager;
   private final StorageContainerManager scmManager;
 
   public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -210,7 +208,6 @@ public class SCMNodeManager
 
     registerMXBean();
 
-    this.nodePoolManager = new SCMNodePoolManager(conf);
     this.scmManager = scmManager;
   }
 
@@ -682,7 +679,6 @@ public class SCMNodeManager
   @Override
   public void close() throws IOException {
     unregisterMXBean();
-    nodePoolManager.close();
     executorService.shutdown();
     try {
       if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -760,20 +756,6 @@ public class SCMNodeManager
       LOG.info("Leaving startup chill mode.");
     }
 
-    // TODO: define node pool policy for non-default node pool.
-    // For now, all nodes are added to the "DefaultNodePool" upon registration
-    // if it has not been added to any node pool yet.
-    try {
-      if (nodePoolManager.getNodePool(datanodeDetails) == null) {
-        nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
-            datanodeDetails);
-      }
-    } catch (IOException e) {
-      // TODO: make sure registration failure is handled correctly.
-      return RegisteredCommand.newBuilder()
-          .setErrorCode(ErrorCode.errorNodeNotPermitted)
-          .build();
-    }
     // Updating Node Report, as registration is successful
     updateNodeStat(datanodeDetails.getUuid(), nodeReport);
     LOG.info("Data node with ID: {} Registered.",
@@ -860,11 +842,6 @@ public class SCMNodeManager
   }
 
   @Override
-  public NodePoolManager getNodePoolManager() {
-    return nodePoolManager;
-  }
-
-  @Override
   public Map<String, Integer> getNodeCount() {
     Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
     for(NodeState state : NodeState.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index faf330e..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMNodePoolManager.class);
-  private static final List<DatanodeDetails> EMPTY_NODE_LIST =
-      new ArrayList<>();
-  private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
-  public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
-  // DB that saves the node to node pool mapping.
-  private MetadataStore nodePoolStore;
-
-  // In-memory node pool to nodes mapping
-  private HashMap<String, Set<DatanodeDetails>> nodePools;
-
-  // Read-write lock for nodepool operations
-  private ReadWriteLock lock;
-
-  /**
-   * Construct SCMNodePoolManager class that manages node to node pool mapping.
-   * @param conf - configuration.
-   * @throws IOException
-   */
-  public SCMNodePoolManager(final OzoneConfiguration conf)
-      throws IOException {
-    final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
-        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-    File metaDir = getOzoneMetaDirPath(conf);
-    String scmMetaDataDir = metaDir.getPath();
-    File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
-    nodePoolStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf)
-        .setDbFile(nodePoolDBPath)
-        .setCacheSize(cacheSize * OzoneConsts.MB)
-        .build();
-    nodePools = new HashMap<>();
-    lock = new ReentrantReadWriteLock();
-    init();
-  }
-
-  /**
-   * Initialize the in-memory store based on persist store from level db.
-   * No lock is needed as init() is only invoked by constructor.
-   * @throws SCMException
-   */
-  private void init() throws SCMException {
-    try {
-      nodePoolStore.iterate(null, (key, value) -> {
-        try {
-          DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
-              HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
-          String poolName = DFSUtil.bytes2String(value);
-
-          Set<DatanodeDetails> nodePool = null;
-          if (nodePools.containsKey(poolName)) {
-            nodePool = nodePools.get(poolName);
-          } else {
-            nodePool = new HashSet<>();
-            nodePools.put(poolName, nodePool);
-          }
-          nodePool.add(nodeId);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding node: {} to node pool: {}",
-                nodeId, poolName);
-          }
-        } catch (IOException e) {
-          LOG.warn("Can't add a datanode to node pool, continue next...");
-        }
-        return true;
-      });
-    } catch (IOException e) {
-      LOG.error("Loading node pool error " + e);
-      throw new SCMException("Failed to load node pool",
-          FAILED_TO_LOAD_NODEPOOL);
-    }
-  }
-
-  /**
-   * Add a datanode to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - name of the datanode.
-   */
-  @Override
-  public void addNode(final String pool, final DatanodeDetails node)
-      throws IOException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // add to the persistent store
-      nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
-          DFSUtil.string2Bytes(pool));
-
-      // add to the in-memory store
-      Set<DatanodeDetails> nodePool = null;
-      if (nodePools.containsKey(pool)) {
-        nodePool = nodePools.get(pool);
-      } else {
-        nodePool = new HashSet<DatanodeDetails>();
-        nodePools.put(pool, nodePool);
-      }
-      nodePool.add(node);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Remove a datanode from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - datanode id.
-   * @throws SCMException
-   */
-  @Override
-  public void removeNode(final String pool, final DatanodeDetails node)
-      throws SCMException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // Remove from the persistent store
-      byte[] kName = node.getProtoBufMessage().toByteArray();
-      byte[] kData = nodePoolStore.get(kName);
-      if (kData == null) {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);
-      }
-      nodePoolStore.delete(kName);
-
-      // Remove from the in-memory store
-      if (nodePools.containsKey(pool)) {
-        Set<DatanodeDetails> nodePool = nodePools.get(pool);
-        nodePool.remove(node);
-      } else {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);
-      }
-    } catch (IOException e) {
-      throw new SCMException("Failed to remove node " + node.toString()
-          + " from node pool " + pool, e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Get all the node pools.
-   * @return all the node pools.
-   */
-  @Override
-  public List<String> getNodePools() {
-    lock.readLock().lock();
-    try {
-      if (!nodePools.isEmpty()) {
-        return nodePools.keySet().stream().collect(Collectors.toList());
-      } else {
-        return EMPTY_NODEPOOL_LIST;
-      }
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get all datanodes of a specific node pool.
-   * @param pool - name of the node pool.
-   * @return all datanodes of the specified node pool.
-   */
-  @Override
-  public List<DatanodeDetails> getNodes(final String pool) {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    if (nodePools.containsKey(pool)) {
-      return nodePools.get(pool).stream().collect(Collectors.toList());
-    } else {
-      return EMPTY_NODE_LIST;
-    }
-  }
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeDetails - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   * TODO: Put this in a in-memory map if performance is an issue.
-   */
-  @Override
-  public String getNodePool(final DatanodeDetails datanodeDetails)
-      throws SCMException {
-    Preconditions.checkNotNull(datanodeDetails, "node is null");
-    try {
-      byte[] result = nodePoolStore.get(
-          datanodeDetails.getProtoBufMessage().toByteArray());
-      return result == null ? null : DFSUtil.bytes2String(result);
-    } catch (IOException e) {
-      throw new SCMException("Failed to get node pool for node "
-          + datanodeDetails.toString(), e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    }
-  }
-
-  /**
-   * Close node pool level db store.
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    nodePoolStore.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 8c59462..80b5d6e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.container;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -273,11 +272,6 @@ public class MockNodeManager implements NodeManager {
     return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
   }
 
-  @Override
-  public NodePoolManager getNodePoolManager() {
-    return Mockito.mock(NodePoolManager.class);
-  }
-
   /**
    * Used for testing.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
deleted file mode 100644
index 8f412de..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for SCM node pool manager.
- */
-public class TestSCMNodePoolManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSCMNodePoolManager.class);
-
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  private final File testDir = PathUtils.getTestDir(
-      TestSCMNodePoolManager.class);
-
-  SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
-      throws IOException {
-    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
-        testDir.getAbsolutePath());
-    conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
-        SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
-    return new SCMNodePoolManager(conf);
-  }
-
-  /**
-   * Test default node pool.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testDefaultNodePool() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    try {
-      final String defaultPool = "DefaultPool";
-      NodePoolManager npMgr = createNodePoolManager(conf);
-
-      final int nodeCount = 4;
-      final List<DatanodeDetails> nodes = TestUtils
-          .getListOfDatanodeDetails(nodeCount);
-      assertEquals(0, npMgr.getNodePools().size());
-      for (DatanodeDetails node: nodes) {
-        npMgr.addNode(defaultPool, node);
-      }
-      List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-      assertEquals(nodeCount, nodesRetrieved.size());
-      assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-
-      DatanodeDetails nodeRemoved = nodes.remove(2);
-      npMgr.removeNode(defaultPool, nodeRemoved);
-      List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
-      assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
-
-      List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
-      assertEquals(0, nonExistSet.size());
-    } finally {
-      FileUtil.fullyDelete(testDir);
-    }
-  }
-
-
-  /**
-   * Test default node pool reload.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testDefaultNodePoolReload() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    final String defaultPool = "DefaultPool";
-    final int nodeCount = 4;
-    final List<DatanodeDetails> nodes = TestUtils
-        .getListOfDatanodeDetails(nodeCount);
-
-    try {
-      try {
-        SCMNodePoolManager npMgr = createNodePoolManager(conf);
-        assertEquals(0, npMgr.getNodePools().size());
-        for (DatanodeDetails node : nodes) {
-          npMgr.addNode(defaultPool, node);
-        }
-        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-        assertEquals(nodeCount, nodesRetrieved.size());
-        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-        npMgr.close();
-      } finally {
-        LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
-            " and close.");
-      }
-
-      // try reload with a new NodePoolManager instance
-      try {
-        SCMNodePoolManager npMgr = createNodePoolManager(conf);
-        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-        assertEquals(nodeCount, nodesRetrieved.size());
-        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-      } finally {
-        LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
-      }
-    } finally {
-      FileUtil.fullyDelete(testDir);
-    }
-  }
-
-  /**
-   * Compare and verify that two datanode lists are equal.
-   * @param list1 - datanode list 1.
-   * @param list2 - datanode list 2.
-   */
-  private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
-      List<DatanodeDetails> list2) {
-    assertEquals(list1.size(), list2.size());
-    Collections.sort(list1);
-    Collections.sort(list2);
-    assertTrue(ListUtils.isEqualList(list1, list2));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 072d821..1a4dcd7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.CommandQueue;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -201,10 +200,6 @@ public class ReplicationNodeManagerMock implements NodeManager {
     return null;
   }
 
-  @Override
-  public NodePoolManager getNodePoolManager() {
-    return Mockito.mock(NodePoolManager.class);
-  }
 
   /**
    * Wait for the heartbeat is processed by NodeManager.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
deleted file mode 100644
index ffcd752..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.testutils;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Pool Manager replication mock.
- */
-public class ReplicationNodePoolManagerMock implements NodePoolManager {
-
-  private final Map<DatanodeDetails, String> nodeMemberShip;
-
-  /**
-   * A node pool manager for testing.
-   */
-  public ReplicationNodePoolManagerMock() {
-    nodeMemberShip = new HashMap<>();
-  }
-
-  /**
-   * Add a node to a node pool.
-   *
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   */
-  @Override
-  public void addNode(String pool, DatanodeDetails node) {
-    nodeMemberShip.put(node, pool);
-  }
-
-  /**
-   * Remove a node from a node pool.
-   *
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   * @throws SCMException
-   */
-  @Override
-  public void removeNode(String pool, DatanodeDetails node)
-      throws SCMException {
-    nodeMemberShip.remove(node);
-
-  }
-
-  /**
-   * Get a list of known node pools.
-   *
-   * @return a list of known node pool names or an empty list if not node pool
-   * is defined.
-   */
-  @Override
-  public List<String> getNodePools() {
-    Set<String> poolSet = new HashSet<>();
-    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
-      poolSet.add(entry.getValue());
-    }
-    return new ArrayList<>(poolSet);
-
-  }
-
-  /**
-   * Get all nodes of a node pool given the name of the node pool.
-   *
-   * @param pool - name of the node pool.
-   * @return a list of datanode ids or an empty list if the node pool was not
-   * found.
-   */
-  @Override
-  public List<DatanodeDetails> getNodes(String pool) {
-    Set<DatanodeDetails> datanodeSet = new HashSet<>();
-    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
-      if (entry.getValue().equals(pool)) {
-        datanodeSet.add(entry.getKey());
-      }
-    }
-    return new ArrayList<>(datanodeSet);
-  }
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   *
-   * @param datanodeDetails DatanodeDetails.
-   * @return node pool name if it has been assigned. null if the node has not
-   * been assigned to any node pool yet.
-   */
-  @Override
-  public String getNodePool(DatanodeDetails datanodeDetails) {
-    return nodeMemberShip.get(datanodeDetails);
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated
-   * with it. If the stream is already closed then invoking this
-   * method has no effect.
-   * <p>
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the
-   * close may fail require careful attention. It is strongly advised
-   * to relinquish the underlying resources and to internally
-   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
-   * the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 4d70af8..b4ed2b1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,12 +51,9 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.UUID;
 
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * This class tests the CLI that transforms container into SQLite DB files.
@@ -177,34 +174,6 @@ public class TestContainerSQLCli {
   }
 
   @Test
-  public void testConvertNodepoolDB() throws Exception {
-    String dbOutPath = GenericTestUtils.getTempPath(
-        UUID.randomUUID() + "/out_sql.db");
-    String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
-    String dbPath = dbRootPath + "/" + NODEPOOL_DB;
-    String[] args = {"-p", dbPath, "-o", dbOutPath};
-
-    cli.run(args);
-
-    // verify the sqlite db
-    HashMap<String, String> expectedPool = new HashMap<>();
-    for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
-      expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
-    }
-    Connection conn = connectDB(dbOutPath);
-    String sql = "SELECT * FROM nodePool";
-    ResultSet rs = executeQuery(conn, sql);
-    while(rs.next()) {
-      String datanodeUUID = rs.getString("datanodeUUID");
-      String poolName = rs.getString("poolName");
-      assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
-    }
-    assertEquals(0, expectedPool.size());
-
-    Files.delete(Paths.get(dbOutPath));
-  }
-
-  @Test
   public void testConvertContainerDB() throws Exception {
     String dbOutPath = GenericTestUtils.getTempPath(
         UUID.randomUUID() + "/out_sql.db");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[04/50] [abbrv] hadoop git commit: YARN-8184. Too many metrics if containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator. Contributed by Yufei Gu

Posted by gi...@apache.org.
YARN-8184. Too many metrics if containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator. Contributed by Yufei Gu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cdce86d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cdce86d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cdce86d

Branch: refs/heads/HADOOP-15461
Commit: 1cdce86d33d4b73ba6dd4136c966eb7e822b6f36
Parents: ae05562
Author: Yufei Gu <yu...@apple.com>
Authored: Fri Jun 22 14:02:32 2018 -0700
Committer: Yufei Gu <yu...@apple.com>
Committed: Fri Jun 22 14:03:55 2018 -0700

----------------------------------------------------------------------
 .../containermanager/localizer/ContainerLocalizer.java          | 5 +----
 .../containermanager/localizer/ResourceLocalizationService.java | 5 +----
 2 files changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 6a384ae..c034369 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -132,10 +132,7 @@ public class ContainerLocalizer {
     this.recordFactory = recordFactory;
     this.conf = new YarnConfiguration();
     this.diskValidator = DiskValidatorFactory.getInstance(
-        conf.get(YarnConfiguration.DISK_VALIDATOR,
-            YarnConfiguration.DEFAULT_DISK_VALIDATOR));
-    LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
-        " is loaded.");
+        YarnConfiguration.DEFAULT_DISK_VALIDATOR);
     this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
     this.pendingResources = new HashMap<LocalResource,Future<Path>>();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index ddae2ae..3f0a6fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -264,10 +264,7 @@ public class ResourceLocalizationService extends CompositeService
     }
 
     diskValidator = DiskValidatorFactory.getInstance(
-        conf.get(YarnConfiguration.DISK_VALIDATOR,
-            YarnConfiguration.DEFAULT_DISK_VALIDATOR));
-    LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
-        " is loaded.");
+        YarnConfiguration.DEFAULT_DISK_VALIDATOR);
     cacheTargetSize =
       conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
     cacheCleanupPeriod =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[16/50] [abbrv] hadoop git commit: HADOOP-15550. Avoid static initialization of ObjectMappers

Posted by gi...@apache.org.
HADOOP-15550. Avoid static initialization of ObjectMappers


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a3c6e9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a3c6e9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a3c6e9c

Branch: refs/heads/HADOOP-15461
Commit: 7a3c6e9c3cd9ffdc71946fd12f5c3d59718c4939
Parents: c687a66
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Jun 25 15:36:45 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Jun 25 15:36:45 2018 -0700

----------------------------------------------------------------------
 .../crypto/key/kms/KMSClientProvider.java       |  7 ++----
 .../web/DelegationTokenAuthenticator.java       |  8 ++-----
 .../apache/hadoop/util/HttpExceptionUtils.java  | 12 ++--------
 .../apache/hadoop/util/JsonSerialization.java   | 24 ++++++++++++++++++++
 .../crypto/key/kms/server/KMSJSONWriter.java    |  6 ++---
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      |  7 ++----
 ...onfRefreshTokenBasedAccessTokenProvider.java |  8 +++----
 .../CredentialBasedAccessTokenProvider.java     |  8 +++----
 .../apache/hadoop/mapreduce/JobSubmitter.java   |  8 +++----
 .../hadoop/fs/azure/security/JsonUtils.java     |  4 ++--
 10 files changed, 45 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index edbf897..7b46075 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
 import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.KMSUtil;
 import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
@@ -79,7 +80,6 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
 import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
@@ -132,9 +132,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
 
   private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
 
-  private static final ObjectWriter WRITER =
-      new ObjectMapper().writerWithDefaultPrettyPrinter();
-
   private final Text dtService;
 
   // Allow fallback to default kms server port 9600 for certain tests that do
@@ -237,7 +234,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   private static void writeJson(Object obj, OutputStream os)
       throws IOException {
     Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
-    WRITER.writeValue(writer, obj);
+    JsonSerialization.writer().writeValue(writer, obj);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 617773b..0ae2af3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -17,8 +17,6 @@
  */
 package org.apache.hadoop.security.token.delegation.web;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.net.NetUtils;
@@ -31,6 +29,7 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,9 +55,6 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
   private static final String CONTENT_TYPE = "Content-Type";
   private static final String APPLICATION_JSON_MIME = "application/json";
 
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
-
   private static final String HTTP_GET = "GET";
   private static final String HTTP_PUT = "PUT";
 
@@ -328,7 +324,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
         if (contentType != null &&
             contentType.contains(APPLICATION_JSON_MIME)) {
           try {
-            ret = READER.readValue(conn.getInputStream());
+            ret = JsonSerialization.mapReader().readValue(conn.getInputStream());
           } catch (Exception ex) {
             throw new AuthenticationException(String.format(
                 "'%s' did not handle the '%s' delegation token operation: %s",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index cdb8112..50be1c3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.util;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -56,11 +53,6 @@ public class HttpExceptionUtils {
 
   private static final String ENTER = System.getProperty("line.separator");
 
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
-  private static final ObjectWriter WRITER =
-      new ObjectMapper().writerWithDefaultPrettyPrinter();
-
   /**
    * Creates a HTTP servlet response serializing the exception in it as JSON.
    *
@@ -82,7 +74,7 @@ public class HttpExceptionUtils {
     Map<String, Object> jsonResponse = new LinkedHashMap<String, Object>();
     jsonResponse.put(ERROR_JSON, json);
     Writer writer = response.getWriter();
-    WRITER.writeValue(writer, jsonResponse);
+    JsonSerialization.writer().writeValue(writer, jsonResponse);
     writer.flush();
   }
 
@@ -150,7 +142,7 @@ public class HttpExceptionUtils {
       InputStream es = null;
       try {
         es = conn.getErrorStream();
-        Map json = READER.readValue(es);
+        Map json = JsonSerialization.mapReader().readValue(es);
         json = (Map) json.get(ERROR_JSON);
         String exClass = (String) json.get(ERROR_CLASSNAME_JSON);
         String exMsg = (String) json.get(ERROR_MESSAGE_JSON);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index 86c4df6..cbc8560 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -25,14 +25,18 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Map;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.google.common.base.Preconditions;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,6 +69,26 @@ public class JsonSerialization<T> {
   private final Class<T> classType;
   private final ObjectMapper mapper;
 
+  private static final ObjectWriter WRITER =
+      new ObjectMapper().writerWithDefaultPrettyPrinter();
+
+  private static final ObjectReader MAP_READER =
+      new ObjectMapper().readerFor(Map.class);
+
+  /**
+   * @return an ObjectWriter which pretty-prints its output
+   */
+  public static ObjectWriter writer() {
+    return WRITER;
+  }
+
+  /**
+   * @return an ObjectReader which returns simple Maps.
+   */
+  public static ObjectReader mapReader() {
+    return MAP_READER;
+  }
+
   /**
    * Create an instance bound to a specific type.
    * @param classType class to marshall

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
index f826572..b9b8d9c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -17,10 +17,9 @@
  */
 package org.apache.hadoop.crypto.key.kms.server;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.JsonSerialization;
 
 import javax.ws.rs.Produces;
 import javax.ws.rs.WebApplicationException;
@@ -67,8 +66,7 @@ public class KMSJSONWriter implements MessageBodyWriter<Object> {
       OutputStream outputStream) throws IOException, WebApplicationException {
     Writer writer = new OutputStreamWriter(outputStream, Charset
         .forName("UTF-8"));
-    ObjectMapper jsonMapper = new ObjectMapper();
-    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
+    JsonSerialization.writer().writeValue(writer, obj);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 673acd6..ec60a18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -56,8 +56,6 @@ import java.util.concurrent.TimeUnit;
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.BoundedInputStream;
 import org.apache.hadoop.conf.Configuration;
@@ -121,6 +119,7 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.TokenSelector;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.KMSUtil;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.StringUtils;
@@ -172,8 +171,6 @@ public class WebHdfsFileSystem extends FileSystem
   private boolean disallowFallbackToInsecureCluster;
   private String restCsrfCustomHeader;
   private Set<String> restCsrfMethodsToIgnore;
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
 
   private DFSOpsCountStatistics storageStatistics;
 
@@ -476,7 +473,7 @@ public class WebHdfsFileSystem extends FileSystem
               + "\" (parsed=\"" + parsed + "\")");
         }
       }
-      return READER.readValue(in);
+      return JsonSerialization.mapReader().readValue(in);
     } finally {
       in.close();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
index c6ebdd6..3e3fbfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import com.squareup.okhttp.OkHttpClient;
 import com.squareup.okhttp.Request;
 import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
 import org.apache.http.HttpStatus;
 
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
 @InterfaceStability.Evolving
 public class ConfRefreshTokenBasedAccessTokenProvider
     extends AccessTokenProvider {
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
 
   public static final String OAUTH_REFRESH_TOKEN_KEY
       = "dfs.webhdfs.oauth2.refresh.token";
@@ -129,7 +126,8 @@ public class ConfRefreshTokenBasedAccessTokenProvider
             + responseBody.code() + ", text = " + responseBody.toString());
       }
 
-      Map<?, ?> response = READER.readValue(responseBody.body().string());
+      Map<?, ?> response = JsonSerialization.mapReader().readValue(
+          responseBody.body().string());
 
       String newExpiresIn = response.get(EXPIRES_IN).toString();
       accessTokenTimer.setExpiresIn(newExpiresIn);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
index 5c629e0..bfd7055 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
  */
 package org.apache.hadoop.hdfs.web.oauth2;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import com.squareup.okhttp.OkHttpClient;
 import com.squareup.okhttp.Request;
 import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.Timer;
 import org.apache.http.HttpStatus;
 
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
 @InterfaceStability.Evolving
 public abstract class CredentialBasedAccessTokenProvider
     extends AccessTokenProvider {
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
 
   public static final String OAUTH_CREDENTIAL_KEY
       = "dfs.webhdfs.oauth2.credential";
@@ -123,7 +120,8 @@ public abstract class CredentialBasedAccessTokenProvider
             + responseBody.code() + ", text = " + responseBody.toString());
       }
 
-      Map<?, ?> response = READER.readValue(responseBody.body().string());
+      Map<?, ?> response = JsonSerialization.mapReader().readValue(
+          responseBody.body().string());
 
       String newExpiresIn = response.get(EXPIRES_IN).toString();
       timer.setExpiresIn(newExpiresIn);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 246986f..ebf9341 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -34,8 +34,6 @@ import javax.crypto.SecretKey;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
@@ -61,6 +59,7 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.JsonSerialization;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.yarn.api.records.ReservationId;
 
@@ -71,8 +70,6 @@ import com.google.common.base.Charsets;
 class JobSubmitter {
   protected static final Logger LOG =
       LoggerFactory.getLogger(JobSubmitter.class);
-  private static final ObjectReader READER =
-      new ObjectMapper().readerFor(Map.class);
   private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
   private static final int SHUFFLE_KEY_LENGTH = 64;
   private FileSystem jtFs;
@@ -406,7 +403,8 @@ class JobSubmitter {
 
       try {
         // read JSON
-        Map<String, String> nm = READER.readValue(new File(localFileName));
+        Map<String, String> nm = JsonSerialization.mapReader().readValue(
+            new File(localFileName));
 
         for(Map.Entry<String, String> ent: nm.entrySet()) {
           credentials.addSecretKey(new Text(ent.getKey()), ent.getValue()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
index 20dd470..9c40325 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.azure.security;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.util.JsonSerialization;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,8 +38,7 @@ public final class JsonUtils {
 
   public static Map<?, ?> parse(final String jsonString) throws IOException {
     try {
-      ObjectMapper mapper = new ObjectMapper();
-      return mapper.readerFor(Map.class).readValue(jsonString);
+      return JsonSerialization.mapReader().readValue(jsonString);
     } catch (Exception e) {
       LOG.debug("JSON Parsing exception: {} while parsing {}", e.getMessage(),
           jsonString);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[36/50] [abbrv] hadoop git commit: HADOOP-14313. Replace/improve Hadoop's byte[] comparator. Contributed by Vikas Vishwakarma.

Posted by gi...@apache.org.
HADOOP-14313. Replace/improve Hadoop's byte[] comparator. Contributed by Vikas Vishwakarma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddbff7c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddbff7c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddbff7c8

Branch: refs/heads/HADOOP-15461
Commit: ddbff7c8d3f1851e5c5fa9bc33637e859d7d8ccf
Parents: 2b2399d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jun 28 14:58:40 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Jun 28 14:58:40 2018 +0900

----------------------------------------------------------------------
 NOTICE.txt                                      |  8 ++++
 .../apache/hadoop/io/FastByteComparisons.java   | 44 ++++++++------------
 2 files changed, 25 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddbff7c8/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 95a670d..a53f13c 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -196,6 +196,14 @@ by Google Inc, which can be obtained at:
   * HOMEPAGE:
     * http://code.google.com/p/snappy/
 
+This product contains a modified portion of UnsignedBytes LexicographicalComparator
+from Guava v21 project by Google Inc, which can be obtained at:
+
+  * LICENSE:
+    * license/COPYING (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/google/guava
+
 This product optionally depends on 'JBoss Marshalling', an alternative Java
 serialization API, which can be obtained at:
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddbff7c8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
index a2903f8..5af6602 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
@@ -26,7 +26,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import sun.misc.Unsafe;
 
-import com.google.common.primitives.Longs;
 import com.google.common.primitives.UnsignedBytes;
 
 /**
@@ -195,52 +194,43 @@ abstract class FastByteComparisons {
             length1 == length2) {
           return 0;
         }
+        final int stride = 8;
         int minLength = Math.min(length1, length2);
-        int minWords = minLength / Longs.BYTES;
+        int strideLimit = minLength & ~(stride - 1);
         int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
         int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+        int i;
 
         /*
          * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
          * time is no slower than comparing 4 bytes at a time even on 32-bit.
          * On the other hand, it is substantially faster on 64-bit.
          */
-        for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
+        for (i = 0; i < strideLimit; i += stride) {
           long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
           long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
-          long diff = lw ^ rw;
 
-          if (diff != 0) {
+          if (lw != rw) {
             if (!littleEndian) {
               return lessThanUnsigned(lw, rw) ? -1 : 1;
             }
 
-            // Use binary search
-            int n = 0;
-            int y;
-            int x = (int) diff;
-            if (x == 0) {
-              x = (int) (diff >>> 32);
-              n = 32;
-            }
-
-            y = x << 16;
-            if (y == 0) {
-              n += 16;
-            } else {
-              x = y;
-            }
-
-            y = x << 8;
-            if (y == 0) {
-              n += 8;
-            }
-            return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+            /*
+             * We want to compare only the first index where left[index] !=
+             * right[index]. This corresponds to the least significant nonzero
+             * byte in lw ^ rw, since lw and rw are little-endian.
+             * Long.numberOfTrailingZeros(diff) tells us the least significant
+             * nonzero bit, and zeroing out the first three bits of L.nTZ gives
+             * us the shift to get that least significant nonzero byte. This
+             * comparison logic is based on UnsignedBytes from Guava v21
+             */
+            int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7;
+            return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF));
           }
         }
 
         // The epilogue to cover the last (minLength % 8) elements.
-        for (int i = minWords * Longs.BYTES; i < minLength; i++) {
+        for (; i < minLength; i++) {
           int result = UnsignedBytes.compare(
               buffer1[offset1 + i],
               buffer2[offset2 + i]);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[13/50] [abbrv] hadoop git commit: YARN-8457. Compilation is broken with -Pyarn-ui.

Posted by gi...@apache.org.
YARN-8457. Compilation is broken with -Pyarn-ui.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ffe68a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ffe68a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ffe68a6

Branch: refs/heads/HADOOP-15461
Commit: 4ffe68a6f70ce01a5654da8991b4cdb35ae0bf1f
Parents: abc3e4b
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Jun 25 10:38:03 2018 -0700
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Jun 25 10:38:03 2018 -0700

----------------------------------------------------------------------
 .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc           | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ffe68a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
index 959e169..daf4462 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
@@ -1,4 +1,5 @@
 {
   "directory": "bower_components",
-  "analytics": false
+  "analytics": false,
+  "registry": "https://registry.bower.io"
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[50/50] [abbrv] hadoop git commit: HADOOP-15522. Deprecate Shell#ReadLink by using native java code. Contributed by Giovanni Matteo Fumarola.

Posted by gi...@apache.org.
HADOOP-15522. Deprecate Shell#ReadLink by using native java code. Contributed by Giovanni Matteo Fumarola.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75f23cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75f23cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75f23cbf

Branch: refs/heads/HADOOP-15461
Commit: 75f23cbf655c36d255d5961e03af9a8f86414404
Parents: 32372ea
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jun 11 13:14:34 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 13:08:57 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/FileUtil.java     | 21 +++++++++++---------
 .../main/java/org/apache/hadoop/util/Shell.java |  8 +++++++-
 2 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/75f23cbf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 61cb8d2..f3b5d58 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -196,22 +196,25 @@ public class FileUtil {
    *         a symlink.
    */
   public static String readLink(File f) {
-    /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
-     * use getCanonicalPath in File to get the target of the symlink but that
-     * does not indicate if the given path refers to a symlink.
-     */
 
     if (f == null) {
       LOG.warn("Can not read a null symLink");
       return "";
     }
 
-    try {
-      return Shell.execCommand(
-          Shell.getReadlinkCommand(f.toString())).trim();
-    } catch (IOException x) {
-      return "";
+    if (Files.isSymbolicLink(f.toPath())) {
+      java.nio.file.Path p = null;
+      try {
+        p = Files.readSymbolicLink(f.toPath());
+      } catch (Exception e) {
+        LOG.warn("Exception while reading the symbolic link "
+            + f.getAbsolutePath() + ". Exception= " + e.getMessage());
+        return "";
+      }
+      return p.toAbsolutePath().toString();
     }
+    LOG.warn("The file " + f.getAbsolutePath() + " is not a symbolic link.");
+    return "";
   }
 
   /*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/75f23cbf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index e902af0..691df63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -309,7 +309,13 @@ public abstract class Shell {
        : new String[] { "ln", "-s", target, link };
   }
 
-  /** Return a command to read the target of the a symbolic link. */
+  /**
+   * Return a command to read the target of the a symbolic link.
+   *
+   * Deprecated and likely to be deleted in the near future. Please use
+   * FileUtil.symlink().
+   */
+  @Deprecated
   public static String[] getReadlinkCommand(String link) {
     return WINDOWS ?
         new String[] { getWinUtilsPath(), "readlink", link }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[22/50] [abbrv] hadoop git commit: YARN-8423. GPU does not get released even though the application gets killed. (Sunil G via wangda)

Posted by gi...@apache.org.
YARN-8423. GPU does not get released even though the application gets killed. (Sunil G via wangda)

Change-Id: I570db7d60f8c6c21762dd618a9207d1107c486a0


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ada8f63d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ada8f63d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ada8f63d

Branch: refs/heads/HADOOP-15461
Commit: ada8f63d0b3739d245300461387b0516dc92ccf9
Parents: 62d83ca
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jun 26 19:25:57 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jun 26 19:25:57 2018 -0700

----------------------------------------------------------------------
 .../containermanager/container/Container.java   |  6 ++
 .../container/ContainerImpl.java                | 11 ++++
 .../resources/gpu/GpuResourceAllocator.java     | 68 +++++++++++++++++++-
 .../resources/gpu/GpuResourceHandlerImpl.java   |  1 -
 .../nodemanager/webapp/MockContainer.java       |  3 +
 5 files changed, 85 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 5d48d84..4912d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -113,4 +113,10 @@ public interface Container extends EventHandler<ContainerEvent> {
   ResourceMappings getResourceMappings();
 
   void sendPauseEvent(String description);
+
+  /**
+   * Verify container is in final states.
+   * @return true/false based on container's state
+   */
+  boolean isContainerInFinalStates();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 0541544..f76e682 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -2223,4 +2223,15 @@ public class ContainerImpl implements Container {
   SlidingWindowRetryPolicy getRetryPolicy() {
     return retryPolicy;
   }
+
+  @Override
+  public boolean isContainerInFinalStates() {
+    ContainerState state = getContainerState();
+    return state == ContainerState.KILLING || state == ContainerState.DONE
+        || state == ContainerState.LOCALIZATION_FAILED
+        || state == ContainerState.CONTAINER_RESOURCES_CLEANINGUP
+        || state == ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL
+        || state == ContainerState.EXITED_WITH_FAILURE
+        || state == ContainerState.EXITED_WITH_SUCCESS;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
index 5bdffc3..81a9655 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
 import org.apache.hadoop.yarn.server.nodemanager.Context;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -36,10 +37,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -54,6 +53,7 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
  */
 public class GpuResourceAllocator {
   final static Log LOG = LogFactory.getLog(GpuResourceAllocator.class);
+  private static final int WAIT_MS_PER_LOOP = 1000;
 
   private Set<GpuDevice> allowedGpuDevices = new TreeSet<>();
   private Map<GpuDevice, ContainerId> usedDevices = new TreeMap<>();
@@ -168,7 +168,44 @@ public class GpuResourceAllocator {
    * @return allocation results.
    * @throws ResourceHandlerException When failed to assign GPUs.
    */
-  public synchronized GpuAllocation assignGpus(Container container)
+  public GpuAllocation assignGpus(Container container)
+      throws ResourceHandlerException {
+    GpuAllocation allocation = internalAssignGpus(container);
+
+    // Wait for a maximum of 120 seconds if no available GPU are there which
+    // are yet to be released.
+    final int timeoutMsecs = 120 * WAIT_MS_PER_LOOP;
+    int timeWaiting = 0;
+    while (allocation == null) {
+      if (timeWaiting >= timeoutMsecs) {
+        break;
+      }
+
+      // Sleep for 1 sec to ensure there are some free GPU devices which are
+      // getting released.
+      try {
+        LOG.info("Container : " + container.getContainerId()
+            + " is waiting for free GPU devices.");
+        Thread.sleep(WAIT_MS_PER_LOOP);
+        timeWaiting += WAIT_MS_PER_LOOP;
+        allocation = internalAssignGpus(container);
+      } catch (InterruptedException e) {
+        // On any interrupt, break the loop and continue execution.
+        break;
+      }
+    }
+
+    if(allocation == null) {
+      String message = "Could not get valid GPU device for container '" +
+          container.getContainerId()
+          + "' as some other containers might not releasing GPUs.";
+      LOG.warn(message);
+      throw new ResourceHandlerException(message);
+    }
+    return allocation;
+  }
+
+  private synchronized GpuAllocation internalAssignGpus(Container container)
       throws ResourceHandlerException {
     Resource requestedResource = container.getResource();
     ContainerId containerId = container.getContainerId();
@@ -176,6 +213,14 @@ public class GpuResourceAllocator {
     // Assign Gpus to container if requested some.
     if (numRequestedGpuDevices > 0) {
       if (numRequestedGpuDevices > getAvailableGpus()) {
+        // If there are some devices which are getting released, wait for few
+        // seconds to get it.
+        if (numRequestedGpuDevices <= getReleasingGpus() + getAvailableGpus()) {
+          return null;
+        }
+      }
+
+      if (numRequestedGpuDevices > getAvailableGpus()) {
         throw new ResourceHandlerException(
             getResourceHandlerExceptionMessage(numRequestedGpuDevices,
                 containerId));
@@ -211,6 +256,23 @@ public class GpuResourceAllocator {
     return new GpuAllocation(null, allowedGpuDevices);
   }
 
+  private synchronized long getReleasingGpus() {
+    long releasingGpus = 0;
+    Iterator<Map.Entry<GpuDevice, ContainerId>> iter = usedDevices.entrySet()
+        .iterator();
+    while (iter.hasNext()) {
+      ContainerId containerId = iter.next().getValue();
+      Container container;
+      if ((container = nmContext.getContainers().get(containerId)) != null) {
+        if (container.isContainerInFinalStates()) {
+          releasingGpus = releasingGpus + container.getResource()
+              .getResourceInformation(ResourceInformation.GPU_URI).getValue();
+        }
+      }
+    }
+    return releasingGpus;
+  }
+
   /**
    * Clean up all Gpus assigned to containerId
    * @param containerId containerId

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 587fcb4..1184382 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
 
-import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 325709b..67dfef2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -255,4 +255,7 @@ public class MockContainer implements Container {
   public void sendPauseEvent(String description) {
 
   }
+  @Override public boolean isContainerInFinalStates() {
+    return false;
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[39/50] [abbrv] hadoop git commit: YARN-8409. Fixed NPE in ActiveStandbyElectorBasedElectorService. Contributed by Chandni Singh

Posted by gi...@apache.org.
YARN-8409.  Fixed NPE in ActiveStandbyElectorBasedElectorService.
            Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/384764cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/384764cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/384764cd

Branch: refs/heads/HADOOP-15461
Commit: 384764cdeac6490bc47fa0eb7b936baa4c0d3230
Parents: d3fa83a
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jun 28 12:39:49 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jun 28 12:39:49 2018 -0400

----------------------------------------------------------------------
 .../apache/hadoop/ha/ActiveStandbyElector.java  |  5 ++++-
 .../apache/hadoop/ha/ZKFailoverController.java  |  2 +-
 .../resourcemanager/TestRMEmbeddedElector.java  | 22 ++++++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index a23fb71..d099ca7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -329,9 +329,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
    * This recursively creates the znode as well as all of its parents.
    */
   public synchronized void ensureParentZNode()
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, KeeperException {
     Preconditions.checkState(!wantToBeInElection,
         "ensureParentZNode() may not be called while in the election");
+    if (zkClient == null) {
+      createConnection();
+    }
 
     String pathParts[] = znodeWorkingDir.split("/");
     Preconditions.checkArgument(pathParts.length >= 1 &&

http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 9295288..f66e3c9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -269,7 +269,7 @@ public abstract class ZKFailoverController {
   }
 
   private int formatZK(boolean force, boolean interactive)
-      throws IOException, InterruptedException {
+      throws IOException, InterruptedException, KeeperException {
     if (elector.parentZNodeExists()) {
       if (!force && (!interactive || !confirmFormat())) {
         return ERR_CODE_FORMAT_DENIED;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
index 9d38149..8c03861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
@@ -22,8 +22,10 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.ClientBaseWithFixes;
 import org.apache.hadoop.ha.ServiceFailedException;
+import org.apache.hadoop.service.ServiceStateException;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -304,6 +306,26 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes {
     verify(as, times(1)).transitionToStandby(any());
   }
 
+  /**
+   * Test that active elector service triggers a fatal RM Event when connection
+   * to ZK fails. YARN-8409
+   */
+  @Test
+  public void testFailureToConnectToZookeeper() throws Exception {
+    stopServer();
+    Configuration myConf = new Configuration(conf);
+    ResourceManager rm = new MockRM(conf);
+
+    ActiveStandbyElectorBasedElectorService ees =
+        new ActiveStandbyElectorBasedElectorService(rm);
+    try {
+      ees.init(myConf);
+      Assert.fail("expect failure to connect to Zookeeper");
+    } catch (ServiceStateException sse) {
+      Assert.assertTrue(sse.getMessage().contains("ConnectionLoss"));
+    }
+  }
+
   private class MockRMWithElector extends MockRM {
     private long delayMs = 0;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[15/50] [abbrv] hadoop git commit: HADOOP-15423. Merge fileCache and dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor Bota.

Posted by gi...@apache.org.
HADOOP-15423. Merge fileCache and dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c687a661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c687a661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c687a661

Branch: refs/heads/HADOOP-15461
Commit: c687a6617d73293019d8d91ac48bbfd2ccca3b40
Parents: a55d6bb
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Jun 25 11:04:34 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Mon Jun 25 14:59:41 2018 -0600

----------------------------------------------------------------------
 .../fs/s3a/s3guard/LocalMetadataEntry.java      |  81 ++++++
 .../fs/s3a/s3guard/LocalMetadataStore.java      | 247 +++++++++++--------
 .../fs/s3a/s3guard/MetadataStoreTestBase.java   |   2 +-
 .../fs/s3a/s3guard/TestLocalMetadataStore.java  |  33 ++-
 4 files changed, 240 insertions(+), 123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
new file mode 100644
index 0000000..6040d67
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import javax.annotation.Nullable;
+
+/**
+ * LocalMetadataEntry is used to store entries in the cache of
+ * LocalMetadataStore. PathMetadata or dirListingMetadata can be null. The
+ * entry is not immutable.
+ */
+public final class LocalMetadataEntry {
+  @Nullable
+  private PathMetadata pathMetadata;
+  @Nullable
+  private DirListingMetadata dirListingMetadata;
+
+  LocalMetadataEntry(PathMetadata pmd){
+    pathMetadata = pmd;
+    dirListingMetadata = null;
+  }
+
+  LocalMetadataEntry(DirListingMetadata dlm){
+    pathMetadata = null;
+    dirListingMetadata = dlm;
+  }
+
+  public PathMetadata getFileMeta() {
+    return pathMetadata;
+  }
+
+  public DirListingMetadata getDirListingMeta() {
+    return dirListingMetadata;
+  }
+
+
+  public boolean hasPathMeta() {
+    return this.pathMetadata != null;
+  }
+
+  public boolean hasDirMeta() {
+    return this.dirListingMetadata != null;
+  }
+
+  public void setPathMetadata(PathMetadata pathMetadata) {
+    this.pathMetadata = pathMetadata;
+  }
+
+  public void setDirListingMetadata(DirListingMetadata dirListingMetadata) {
+    this.dirListingMetadata = dirListingMetadata;
+  }
+
+  @Override public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("LocalMetadataEntry{");
+    if(pathMetadata != null) {
+      sb.append("pathMetadata=" + pathMetadata.getFileStatus().getPath());
+    }
+    if(dirListingMetadata != null){
+      sb.append("; dirListingMetadata=" + dirListingMetadata.getPath());
+    }
+    sb.append("}");
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 95689e1..49981ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -37,13 +37,12 @@ import java.io.IOException;
 import java.net.URI;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 /**
- * This is a local, in-memory, implementation of MetadataStore.
+ * This is a local, in-memory implementation of MetadataStore.
  * This is <i>not</i> a coherent cache across processes.  It is only
  * locally-coherent.
  *
@@ -56,12 +55,12 @@ import java.util.concurrent.TimeUnit;
  * non-recursive removal of non-empty directories.  It is assumed the caller
  * already has to perform these sorts of checks.
  *
- * Contains cache internally with time based eviction.
+ * Contains one cache internally with time based eviction.
  */
 public class LocalMetadataStore implements MetadataStore {
 
   public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
-  public static final int DEFAULT_MAX_RECORDS = 128;
+  public static final int DEFAULT_MAX_RECORDS = 256;
   public static final int DEFAULT_CACHE_ENTRY_TTL_MSEC = 10 * 1000;
 
   /**
@@ -79,11 +78,8 @@ public class LocalMetadataStore implements MetadataStore {
   public static final String CONF_CACHE_ENTRY_TTL =
       "fs.metadatastore.local.ttl";
 
-  /** Contains directories and files. */
-  private Cache<Path, PathMetadata> fileCache;
-
-  /** Contains directory listings. */
-  private Cache<Path, DirListingMetadata> dirCache;
+  /** Contains directory and file listings. */
+  private Cache<Path, LocalMetadataEntry> localCache;
 
   private FileSystem fs;
   /* Null iff this FS does not have an associated URI host. */
@@ -116,8 +112,7 @@ public class LocalMetadataStore implements MetadataStore {
       builder.expireAfterAccess(ttl, TimeUnit.MILLISECONDS);
     }
 
-    fileCache = builder.build();
-    dirCache = builder.build();
+    localCache = builder.build();
   }
 
   @Override
@@ -155,8 +150,7 @@ public class LocalMetadataStore implements MetadataStore {
 
     if (recursive) {
       // Remove all entries that have this dir as path prefix.
-      deleteEntryByAncestor(path, dirCache, tombstone);
-      deleteEntryByAncestor(path, fileCache, tombstone);
+      deleteEntryByAncestor(path, localCache, tombstone);
     }
   }
 
@@ -170,7 +164,7 @@ public class LocalMetadataStore implements MetadataStore {
       throws IOException {
     Path path = standardize(p);
     synchronized (this) {
-      PathMetadata m = fileCache.getIfPresent(path);
+      PathMetadata m = getFileMeta(path);
 
       if (wantEmptyDirectoryFlag && m != null &&
           m.getFileStatus().isDirectory()) {
@@ -191,15 +185,15 @@ public class LocalMetadataStore implements MetadataStore {
    * @return TRUE / FALSE if known empty / not-empty, UNKNOWN otherwise.
    */
   private Tristate isEmptyDirectory(Path p) {
-    DirListingMetadata dirMeta = dirCache.getIfPresent(p);
-    return dirMeta.withoutTombstones().isEmpty();
+    DirListingMetadata dlm = getDirListingMeta(p);
+    return dlm.withoutTombstones().isEmpty();
   }
 
   @Override
   public synchronized DirListingMetadata listChildren(Path p) throws
       IOException {
     Path path = standardize(p);
-    DirListingMetadata listing = dirCache.getIfPresent(path);
+    DirListingMetadata listing = getDirListingMeta(path);
     if (LOG.isDebugEnabled()) {
       LOG.debug("listChildren({}) -> {}", path,
           listing == null ? "null" : listing.prettyPrint());
@@ -211,6 +205,7 @@ public class LocalMetadataStore implements MetadataStore {
   @Override
   public void move(Collection<Path> pathsToDelete,
       Collection<PathMetadata> pathsToCreate) throws IOException {
+    LOG.info("Move {} to {}", pathsToDelete, pathsToCreate);
 
     Preconditions.checkNotNull(pathsToDelete, "pathsToDelete is null");
     Preconditions.checkNotNull(pathsToCreate, "pathsToCreate is null");
@@ -258,7 +253,12 @@ public class LocalMetadataStore implements MetadataStore {
       if (LOG.isDebugEnabled()) {
         LOG.debug("put {} -> {}", path, meta.prettyPrint());
       }
-      fileCache.put(path, meta);
+      LocalMetadataEntry entry = localCache.getIfPresent(path);
+      if(entry == null){
+        entry = new LocalMetadataEntry(meta);
+      } else {
+        entry.setPathMetadata(meta);
+      }
 
       /* Directory case:
        * We also make sure we have an entry in the dirCache, so subsequent
@@ -271,27 +271,32 @@ public class LocalMetadataStore implements MetadataStore {
        * saving round trips to underlying store for subsequent listStatus()
        */
 
-      if (status.isDirectory()) {
-        DirListingMetadata dir = dirCache.getIfPresent(path);
-        if (dir == null) {
-          dirCache.put(path, new DirListingMetadata(path, DirListingMetadata
-              .EMPTY_DIR, false));
-        }
+      // only create DirListingMetadata if the entry does not have one
+      if (status.isDirectory() && !entry.hasDirMeta()) {
+        DirListingMetadata dlm =
+            new DirListingMetadata(path, DirListingMetadata.EMPTY_DIR, false);
+        entry.setDirListingMetadata(dlm);
       }
+      localCache.put(path, entry);
 
       /* Update cached parent dir. */
       Path parentPath = path.getParent();
       if (parentPath != null) {
-        DirListingMetadata parent = dirCache.getIfPresent(parentPath);
-        if (parent == null) {
-        /* Track this new file's listing in parent.  Parent is not
-         * authoritative, since there may be other items in it we don't know
-         * about. */
-          parent = new DirListingMetadata(parentPath,
-              DirListingMetadata.EMPTY_DIR, false);
-          dirCache.put(parentPath, parent);
+        LocalMetadataEntry parentMeta = localCache.getIfPresent(parentPath);
+        DirListingMetadata parentDirMeta =
+            new DirListingMetadata(parentPath, DirListingMetadata.EMPTY_DIR,
+                false);
+        parentDirMeta.put(status);
+
+        getDirListingMeta(parentPath);
+
+        if (parentMeta == null){
+          localCache.put(parentPath, new LocalMetadataEntry(parentDirMeta));
+        } else if (!parentMeta.hasDirMeta()) {
+          parentMeta.setDirListingMetadata(parentDirMeta);
+        } else {
+          parentMeta.getDirListingMeta().put(status);
         }
-        parent.put(status);
       }
     }
   }
@@ -301,7 +306,13 @@ public class LocalMetadataStore implements MetadataStore {
     if (LOG.isDebugEnabled()) {
       LOG.debug("put dirMeta {}", meta.prettyPrint());
     }
-    dirCache.put(standardize(meta.getPath()), meta);
+    LocalMetadataEntry entry =
+        localCache.getIfPresent(standardize(meta.getPath()));
+    if(entry == null){
+      localCache.put(standardize(meta.getPath()), new LocalMetadataEntry(meta));
+    } else {
+      entry.setDirListingMetadata(meta);
+    }
     put(meta.getListing());
   }
 
@@ -319,8 +330,8 @@ public class LocalMetadataStore implements MetadataStore {
 
   @Override
   public void destroy() throws IOException {
-    if (dirCache != null) {
-      dirCache.invalidateAll();
+    if (localCache != null) {
+      localCache.invalidateAll();
     }
   }
 
@@ -330,42 +341,44 @@ public class LocalMetadataStore implements MetadataStore {
   }
 
   @Override
-  public synchronized void prune(long modTime, String keyPrefix)
-      throws IOException {
-    Iterator<Map.Entry<Path, PathMetadata>> files =
-        fileCache.asMap().entrySet().iterator();
-    while (files.hasNext()) {
-      Map.Entry<Path, PathMetadata> entry = files.next();
-      if (expired(entry.getValue().getFileStatus(), modTime, keyPrefix)) {
-        files.remove();
-      }
-    }
-    Iterator<Map.Entry<Path, DirListingMetadata>> dirs =
-        dirCache.asMap().entrySet().iterator();
-    while (dirs.hasNext()) {
-      Map.Entry<Path, DirListingMetadata> entry = dirs.next();
-      Path path = entry.getKey();
-      DirListingMetadata metadata = entry.getValue();
-      Collection<PathMetadata> oldChildren = metadata.getListing();
-      Collection<PathMetadata> newChildren = new LinkedList<>();
-
-      for (PathMetadata child : oldChildren) {
-        FileStatus status = child.getFileStatus();
-        if (!expired(status, modTime, keyPrefix)) {
-          newChildren.add(child);
-        }
-      }
-      if (newChildren.size() != oldChildren.size()) {
-        dirCache.put(path, new DirListingMetadata(path, newChildren, false));
-        if (!path.isRoot()) {
-          DirListingMetadata parent = null;
-          parent = dirCache.getIfPresent(path.getParent());
-          if (parent != null) {
-            parent.setAuthoritative(false);
+  public synchronized void prune(long modTime, String keyPrefix) {
+    // prune files
+    // filter path_metadata (files), filter expired, remove expired
+    localCache.asMap().entrySet().stream()
+        .filter(entry -> entry.getValue().hasPathMeta())
+        .filter(entry -> expired(
+            entry.getValue().getFileMeta().getFileStatus(), modTime, keyPrefix))
+        .forEach(entry -> localCache.invalidate(entry.getKey()));
+
+
+    // prune dirs
+    // filter DIR_LISTING_METADATA, remove expired, remove authoritative bit
+    localCache.asMap().entrySet().stream()
+        .filter(entry -> entry.getValue().hasDirMeta())
+        .forEach(entry -> {
+          Path path = entry.getKey();
+          DirListingMetadata metadata = entry.getValue().getDirListingMeta();
+          Collection<PathMetadata> oldChildren = metadata.getListing();
+          Collection<PathMetadata> newChildren = new LinkedList<>();
+
+          for (PathMetadata child : oldChildren) {
+            FileStatus status = child.getFileStatus();
+            if (!expired(status, modTime, keyPrefix)) {
+              newChildren.add(child);
+            }
           }
-        }
-      }
-    }
+          if (newChildren.size() != oldChildren.size()) {
+            DirListingMetadata dlm =
+                new DirListingMetadata(path, newChildren, false);
+            localCache.put(path, new LocalMetadataEntry(dlm));
+            if (!path.isRoot()) {
+              DirListingMetadata parent = getDirListingMeta(path.getParent());
+              if (parent != null) {
+                parent.setAuthoritative(false);
+              }
+            }
+          }
+        });
   }
 
   private boolean expired(FileStatus status, long expiry, String keyPrefix) {
@@ -390,31 +403,26 @@ public class LocalMetadataStore implements MetadataStore {
   }
 
   @VisibleForTesting
-  static <T> void deleteEntryByAncestor(Path ancestor, Cache<Path, T> cache,
-                                       boolean tombstone) {
-    for (Iterator<Map.Entry<Path, T>> it = cache.asMap().entrySet().iterator();
-         it.hasNext();) {
-      Map.Entry<Path, T> entry = it.next();
-      Path f = entry.getKey();
-      T meta = entry.getValue();
-      if (isAncestorOf(ancestor, f)) {
-        if (tombstone) {
-          if (meta instanceof PathMetadata) {
-            cache.put(f, (T) PathMetadata.tombstone(f));
-          } else if (meta instanceof DirListingMetadata) {
-            it.remove();
+  static void deleteEntryByAncestor(Path ancestor,
+      Cache<Path, LocalMetadataEntry> cache, boolean tombstone) {
+
+    cache.asMap().entrySet().stream()
+        .filter(entry -> isAncestorOf(ancestor, entry.getKey()))
+        .forEach(entry -> {
+          LocalMetadataEntry meta = entry.getValue();
+          Path path = entry.getKey();
+          if(meta.hasDirMeta()){
+            cache.invalidate(path);
+          } else if(tombstone && meta.hasPathMeta()){
+            meta.setPathMetadata(PathMetadata.tombstone(path));
           } else {
-            throw new IllegalStateException("Unknown type in cache");
+            cache.invalidate(path);
           }
-        } else {
-          it.remove();
-        }
-      }
-    }
+        });
   }
 
   /**
-   * @return true iff 'ancestor' is ancestor dir in path 'f'.
+   * @return true if 'ancestor' is ancestor dir in path 'f'.
    * All paths here are absolute.  Dir does not count as its own ancestor.
    */
   private static boolean isAncestorOf(Path ancestor, Path f) {
@@ -431,27 +439,41 @@ public class LocalMetadataStore implements MetadataStore {
    * lock held.
    */
   private void deleteCacheEntries(Path path, boolean tombstone) {
+    LocalMetadataEntry entry = localCache.getIfPresent(path);
+    // If there's no entry, delete should silently succeed
+    // (based on MetadataStoreTestBase#testDeleteNonExisting)
+    if(entry == null){
+      LOG.warn("Delete: path {} is missing from cache.", path);
+      return;
+    }
 
-    // Remove target file/dir
+    // Remove target file entry
     LOG.debug("delete file entry for {}", path);
-    if (tombstone) {
-      fileCache.put(path, PathMetadata.tombstone(path));
-    } else {
-      fileCache.invalidate(path);
+    if(entry.hasPathMeta()){
+      if (tombstone) {
+        PathMetadata pmd = PathMetadata.tombstone(path);
+        entry.setPathMetadata(pmd);
+      } else {
+        entry.setPathMetadata(null);
+      }
     }
 
-    // Update this and parent dir listing, if any
-
-    /* If this path is a dir, remove its listing */
-    LOG.debug("removing listing of {}", path);
+    // If this path is a dir, remove its listing
+    if(entry.hasDirMeta()) {
+      LOG.debug("removing listing of {}", path);
+      entry.setDirListingMetadata(null);
+    }
 
-    dirCache.invalidate(path);
+    // If the entry is empty (contains no dirMeta or pathMeta) remove it from
+    // the cache.
+    if(!entry.hasDirMeta() && !entry.hasPathMeta()){
+      localCache.invalidate(entry);
+    }
 
     /* Remove this path from parent's dir listing */
     Path parent = path.getParent();
     if (parent != null) {
-      DirListingMetadata dir = null;
-      dir = dirCache.getIfPresent(parent);
+      DirListingMetadata dir = getDirListingMeta(parent);
       if (dir != null) {
         LOG.debug("removing parent's entry for {} ", path);
         if (tombstone) {
@@ -494,4 +516,23 @@ public class LocalMetadataStore implements MetadataStore {
   public void updateParameters(Map<String, String> parameters)
       throws IOException {
   }
+
+  PathMetadata getFileMeta(Path p){
+    LocalMetadataEntry entry = localCache.getIfPresent(p);
+    if(entry != null && entry.hasPathMeta()){
+      return entry.getFileMeta();
+    } else {
+      return null;
+    }
+  }
+
+  DirListingMetadata getDirListingMeta(Path p){
+    LocalMetadataEntry entry = localCache.getIfPresent(p);
+    if(entry != null && entry.hasDirMeta()){
+      return entry.getDirListingMeta();
+    } else {
+      return null;
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 56618cb..5a59400 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -836,7 +836,7 @@ public abstract class MetadataStoreTestBase extends HadoopTestBase {
       throws IOException {
     DirListingMetadata dirMeta = ms.listChildren(strToPath(pathStr));
     if (!allowMissing()) {
-      assertNotNull("Directory " + pathStr + " in cache", dirMeta);
+      assertNotNull("Directory " + pathStr + " is null in cache", dirMeta);
     }
     if (!allowMissing() || dirMeta != null) {
       dirMeta = dirMeta.withoutTombstones();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index 074319f..2ea20b2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.s3a.S3ATestUtils;
  */
 public class TestLocalMetadataStore extends MetadataStoreTestBase {
 
-  private static final String MAX_ENTRIES_STR = "16";
 
   private final static class LocalMSContract extends AbstractMSContract {
 
@@ -48,7 +47,6 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     }
 
     private LocalMSContract(Configuration config) throws IOException {
-      config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
       fs = FileSystem.getLocal(config);
     }
 
@@ -76,8 +74,8 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
   }
 
   @Test
-  public void testClearByAncestor() {
-    Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder().build();
+  public void testClearByAncestor() throws Exception {
+    Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder().build();
 
     // 1. Test paths without scheme/host
     assertClearResult(cache, "", "/", 0);
@@ -122,7 +120,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
 
     final long ttl = t1 + 50; // between t1 and t2
 
-    Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder()
+    Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder()
         .expireAfterWrite(ttl,
             TimeUnit.NANOSECONDS /* nanos to avoid conversions */)
         .ticker(testTicker)
@@ -143,7 +141,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
 
     assertEquals("Cache should contain 3 records before eviction",
         3, cache.size());
-    PathMetadata pm1 = cache.getIfPresent(path1);
+    LocalMetadataEntry pm1 = cache.getIfPresent(path1);
     assertNotNull("PathMetadata should not be null before eviction", pm1);
 
     // set the ticker to a time when timed eviction should occur
@@ -159,7 +157,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     assertNull("PathMetadata should be null after eviction", pm1);
   }
 
-  private static void populateMap(Cache<Path, PathMetadata> cache,
+  private static void populateMap(Cache<Path, LocalMetadataEntry> cache,
       String prefix) {
     populateEntry(cache, new Path(prefix + "/dirA/dirB/"));
     populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC"));
@@ -168,23 +166,20 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
     populateEntry(cache, new Path(prefix + "/dirA/file1"));
   }
 
-  private static void populateEntry(Cache<Path, PathMetadata> cache,
+  private static void populateEntry(Cache<Path, LocalMetadataEntry> cache,
       Path path) {
-    cache.put(path, new PathMetadata(new FileStatus(0, true, 0, 0, 0, path)));
+    FileStatus fileStatus = new FileStatus(0, true, 0, 0, 0, path);
+    cache.put(path, new LocalMetadataEntry(new PathMetadata(fileStatus)));
   }
 
-  private static int sizeOfMap(Cache<Path, PathMetadata> cache) {
-    int count = 0;
-    for (PathMetadata meta : cache.asMap().values()) {
-      if (!meta.isDeleted()) {
-        count++;
-      }
-    }
-    return count;
+  private static long sizeOfMap(Cache<Path, LocalMetadataEntry> cache) {
+    return cache.asMap().values().stream()
+        .filter(entry -> !entry.getFileMeta().isDeleted())
+        .count();
   }
 
-  private static void assertClearResult(Cache<Path, PathMetadata> cache,
-      String prefixStr, String pathStr, int leftoverSize) {
+  private static void assertClearResult(Cache<Path, LocalMetadataEntry> cache,
+      String prefixStr, String pathStr, int leftoverSize) throws IOException {
     populateMap(cache, prefixStr);
     LocalMetadataStore.deleteEntryByAncestor(new Path(prefixStr + pathStr),
         cache, true);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[40/50] [abbrv] hadoop git commit: YARN-8379. Improve balancing resources in already satisfied queues by using Capacity Scheduler preemption. Contributed by Zian Chen.

Posted by gi...@apache.org.
YARN-8379. Improve balancing resources in already satisfied queues by using Capacity Scheduler preemption. Contributed by Zian Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29119430
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29119430
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29119430

Branch: refs/heads/HADOOP-15461
Commit: 291194302cc1a875d6d94ea93cf1184a3f1fc2cc
Parents: 384764c
Author: Sunil G <su...@apache.org>
Authored: Thu Jun 28 10:23:31 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Thu Jun 28 10:23:31 2018 -0700

----------------------------------------------------------------------
 .../AbstractPreemptableResourceCalculator.java  |  21 +-
 .../CapacitySchedulerPreemptionContext.java     |   2 +
 .../CapacitySchedulerPreemptionUtils.java       |  23 +-
 .../capacity/FifoCandidatesSelector.java        |  45 ++--
 .../capacity/IntraQueueCandidatesSelector.java  |   9 +-
 .../capacity/PreemptableResourceCalculator.java |   7 +-
 .../capacity/PreemptionCandidatesSelector.java  |  11 +
 .../ProportionalCapacityPreemptionPolicy.java   | 129 +++++++---
 ...QueuePriorityContainerCandidateSelector.java |  16 +-
 .../ReservedContainerCandidatesSelector.java    |  16 +-
 .../monitor/capacity/TempQueuePerPartition.java |   8 +-
 .../CapacitySchedulerConfiguration.java         |  17 ++
 .../TestPreemptionForQueueWithPriorities.java   |  58 +++++
 ...apacityPreemptionPolicyPreemptToBalance.java | 254 +++++++++++++++++++
 ...TestCapacitySchedulerSurgicalPreemption.java | 111 ++++++++
 15 files changed, 637 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
index 64b3615..5b8360a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
@@ -42,6 +42,7 @@ public class AbstractPreemptableResourceCalculator {
   protected final ResourceCalculator rc;
   protected boolean isReservedPreemptionCandidatesSelector;
   private Resource stepFactor;
+  private boolean allowQueuesBalanceAfterAllQueuesSatisfied;
 
   static class TQComparator implements Comparator<TempQueuePerPartition> {
     private ResourceCalculator rc;
@@ -83,15 +84,28 @@ public class AbstractPreemptableResourceCalculator {
    *          this will be set by different implementation of candidate
    *          selectors, please refer to TempQueuePerPartition#offer for
    *          details.
+   * @param allowQueuesBalanceAfterAllQueuesSatisfied
+   *          Should resources be preempted from an over-served queue when the
+   *          requesting queues are all at or over their guarantees?
+   *          An example is, there're 10 queues under root, guaranteed resource
+   *          of them are all 10%.
+   *          Assume there're two queues are using resources, queueA uses 10%
+   *          queueB uses 90%. For all queues are guaranteed, but it's not fair
+   *          for queueA.
+   *          We wanna make this behavior can be configured. By default it is
+   *          not allowed.
+   *
    */
   public AbstractPreemptableResourceCalculator(
       CapacitySchedulerPreemptionContext preemptionContext,
-      boolean isReservedPreemptionCandidatesSelector) {
+      boolean isReservedPreemptionCandidatesSelector,
+      boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
     context = preemptionContext;
     rc = preemptionContext.getResourceCalculator();
     this.isReservedPreemptionCandidatesSelector =
         isReservedPreemptionCandidatesSelector;
-
+    this.allowQueuesBalanceAfterAllQueuesSatisfied =
+        allowQueuesBalanceAfterAllQueuesSatisfied;
     stepFactor = Resource.newInstance(0, 0);
     for (ResourceInformation ri : stepFactor.getResources()) {
       ri.setValue(1);
@@ -193,7 +207,8 @@ public class AbstractPreemptableResourceCalculator {
         wQavail = Resources.componentwiseMin(wQavail, unassigned);
 
         Resource wQidle = sub.offer(wQavail, rc, totGuarant,
-            isReservedPreemptionCandidatesSelector);
+            isReservedPreemptionCandidatesSelector,
+            allowQueuesBalanceAfterAllQueuesSatisfied);
         Resource wQdone = Resources.subtract(wQavail, wQidle);
 
         if (Resources.greaterThan(rc, totGuarant, wQdone, Resources.none())) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
index 098acdd..7985296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
@@ -70,6 +70,8 @@ public interface CapacitySchedulerPreemptionContext {
 
   float getMaxAllowableLimitForIntraQueuePreemption();
 
+  long getDefaultMaximumKillWaitTimeout();
+
   @Unstable
   IntraQueuePreemptionOrderPolicy getIntraQueuePreemptionOrderPolicy();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
index 690eb02..ed50eff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -151,6 +151,7 @@ public class CapacitySchedulerPreemptionUtils {
       Map<String, Resource> resourceToObtainByPartitions,
       RMContainer rmContainer, Resource clusterResource,
       Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       Resource totalPreemptionAllowed, boolean conservativeDRF) {
     ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId();
 
@@ -218,7 +219,7 @@ public class CapacitySchedulerPreemptionUtils {
       }
 
       // Add to preemptMap
-      addToPreemptMap(preemptMap, attemptId, rmContainer);
+      addToPreemptMap(preemptMap, curCandidates, attemptId, rmContainer);
       return true;
     }
 
@@ -230,15 +231,23 @@ public class CapacitySchedulerPreemptionUtils {
     return context.getScheduler().getSchedulerNode(nodeId).getPartition();
   }
 
-  private static void addToPreemptMap(
+  protected static void addToPreemptMap(
       Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) {
-    Set<RMContainer> set = preemptMap.get(appAttemptId);
-    if (null == set) {
-      set = new HashSet<>();
-      preemptMap.put(appAttemptId, set);
+    Set<RMContainer> setForToPreempt = preemptMap.get(appAttemptId);
+    Set<RMContainer> setForCurCandidates = curCandidates.get(appAttemptId);
+    if (null == setForToPreempt) {
+      setForToPreempt = new HashSet<>();
+      preemptMap.put(appAttemptId, setForToPreempt);
     }
-    set.add(containerToPreempt);
+    setForToPreempt.add(containerToPreempt);
+
+    if (null == setForCurCandidates) {
+      setForCurCandidates = new HashSet<>();
+      curCandidates.put(appAttemptId, setForCurCandidates);
+    }
+    setForCurCandidates.add(containerToPreempt);
   }
 
   private static boolean preemptMapContains(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index 3b2fcbb..c2735f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
 import org.apache.hadoop.yarn.util.resource.Resources;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -42,19 +43,25 @@ public class FifoCandidatesSelector
   private static final Log LOG =
       LogFactory.getLog(FifoCandidatesSelector.class);
   private PreemptableResourceCalculator preemptableAmountCalculator;
+  private boolean allowQueuesBalanceAfterAllQueuesSatisfied;
 
   FifoCandidatesSelector(CapacitySchedulerPreemptionContext preemptionContext,
-      boolean includeReservedResource) {
+      boolean includeReservedResource,
+      boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
     super(preemptionContext);
 
+    this.allowQueuesBalanceAfterAllQueuesSatisfied =
+        allowQueuesBalanceAfterAllQueuesSatisfied;
     preemptableAmountCalculator = new PreemptableResourceCalculator(
-        preemptionContext, includeReservedResource);
+        preemptionContext, includeReservedResource,
+        allowQueuesBalanceAfterAllQueuesSatisfied);
   }
 
   @Override
   public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
       Resource clusterResource, Resource totalPreemptionAllowed) {
+    Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
     // Calculate how much resources we need to preempt
     preemptableAmountCalculator.computeIdealAllocation(clusterResource,
         totalPreemptionAllowed);
@@ -110,7 +117,7 @@ public class FifoCandidatesSelector
               boolean preempted = CapacitySchedulerPreemptionUtils
                   .tryPreemptContainerAndDeductResToObtain(rc,
                       preemptionContext, resToObtainByPartition, c,
-                      clusterResource, selectedCandidates,
+                      clusterResource, selectedCandidates, curCandidates,
                       totalPreemptionAllowed, false);
               if (!preempted) {
                 continue;
@@ -134,7 +141,7 @@ public class FifoCandidatesSelector
 
           preemptFrom(fc, clusterResource, resToObtainByPartition,
               skippedAMContainerlist, skippedAMSize, selectedCandidates,
-              totalPreemptionAllowed);
+              curCandidates, totalPreemptionAllowed);
         }
 
         // Can try preempting AMContainers (still saving atmost
@@ -145,15 +152,15 @@ public class FifoCandidatesSelector
                 leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
                 leafQueue.getMaxAMResourcePerQueuePercent());
 
-        preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
-            resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
-            totalPreemptionAllowed);
+        preemptAMContainers(clusterResource, selectedCandidates, curCandidates,
+            skippedAMContainerlist, resToObtainByPartition, skippedAMSize,
+            maxAMCapacityForThisQueue, totalPreemptionAllowed);
       } finally {
         leafQueue.getReadLock().unlock();
       }
     }
 
-    return selectedCandidates;
+    return curCandidates;
   }
 
   /**
@@ -169,6 +176,7 @@ public class FifoCandidatesSelector
    */
   private void preemptAMContainers(Resource clusterResource,
       Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       List<RMContainer> skippedAMContainerlist,
       Map<String, Resource> resToObtainByPartition, Resource skippedAMSize,
       Resource maxAMCapacityForThisQueue, Resource totalPreemptionAllowed) {
@@ -187,7 +195,7 @@ public class FifoCandidatesSelector
       boolean preempted = CapacitySchedulerPreemptionUtils
           .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
               resToObtainByPartition, c, clusterResource, preemptMap,
-              totalPreemptionAllowed, false);
+              curCandidates, totalPreemptionAllowed, false);
       if (preempted) {
         Resources.subtractFrom(skippedAMSize, c.getAllocatedResource());
       }
@@ -203,6 +211,7 @@ public class FifoCandidatesSelector
       Resource clusterResource, Map<String, Resource> resToObtainByPartition,
       List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
       Map<ApplicationAttemptId, Set<RMContainer>> selectedContainers,
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       Resource totalPreemptionAllowed) {
     ApplicationAttemptId appId = app.getApplicationAttemptId();
 
@@ -219,9 +228,10 @@ public class FifoCandidatesSelector
       }
 
       // Try to preempt this container
-      CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
-          rc, preemptionContext, resToObtainByPartition, c, clusterResource,
-          selectedContainers, totalPreemptionAllowed, false);
+     CapacitySchedulerPreemptionUtils
+          .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
+              resToObtainByPartition, c, clusterResource, selectedContainers,
+              curCandidates, totalPreemptionAllowed, false);
 
       if (!preemptionContext.isObserveOnly()) {
         preemptionContext.getRMContext().getDispatcher().getEventHandler()
@@ -262,9 +272,14 @@ public class FifoCandidatesSelector
       }
 
       // Try to preempt this container
-      CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
-          rc, preemptionContext, resToObtainByPartition, c, clusterResource,
-          selectedContainers, totalPreemptionAllowed, false);
+      CapacitySchedulerPreemptionUtils
+          .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
+              resToObtainByPartition, c, clusterResource, selectedContainers,
+              curCandidates, totalPreemptionAllowed, false);
     }
   }
+
+  public boolean getAllowQueuesBalanceAfterAllQueuesSatisfied() {
+    return allowQueuesBalanceAfterAllQueuesSatisfied;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
index 8ab9507..c52fd95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
@@ -122,7 +122,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
   public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
       Resource clusterResource, Resource totalPreemptedResourceAllowed) {
-
+    Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
     // 1. Calculate the abnormality within each queue one by one.
     computeIntraQueuePreemptionDemand(
         clusterResource, totalPreemptedResourceAllowed, selectedCandidates);
@@ -182,7 +182,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
           leafQueue.getReadLock().lock();
           for (FiCaSchedulerApp app : apps) {
             preemptFromLeastStarvedApp(leafQueue, app, selectedCandidates,
-                clusterResource, totalPreemptedResourceAllowed,
+                curCandidates, clusterResource, totalPreemptedResourceAllowed,
                 resToObtainByPartition, rollingResourceUsagePerUser);
           }
         } finally {
@@ -191,7 +191,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
       }
     }
 
-    return selectedCandidates;
+    return curCandidates;
   }
 
   private void initializeUsageAndUserLimitForCompute(Resource clusterResource,
@@ -211,6 +211,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
   private void preemptFromLeastStarvedApp(LeafQueue leafQueue,
       FiCaSchedulerApp app,
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
       Resource clusterResource, Resource totalPreemptedResourceAllowed,
       Map<String, Resource> resToObtainByPartition,
       Map<String, Resource> rollingResourceUsagePerUser) {
@@ -270,7 +271,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
       boolean ret = CapacitySchedulerPreemptionUtils
           .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
               resToObtainByPartition, c, clusterResource, selectedCandidates,
-              totalPreemptedResourceAllowed, true);
+              curCandidates, totalPreemptedResourceAllowed, true);
 
       // Subtract from respective user's resource usage once a container is
       // selected for preemption.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
index 08d834e..89a015e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
@@ -48,11 +48,14 @@ public class PreemptableResourceCalculator
    * @param isReservedPreemptionCandidatesSelector this will be set by
    * different implementation of candidate selectors, please refer to
    * TempQueuePerPartition#offer for details.
+   * @param allowQueuesBalanceAfterAllQueuesSatisfied
    */
   public PreemptableResourceCalculator(
       CapacitySchedulerPreemptionContext preemptionContext,
-      boolean isReservedPreemptionCandidatesSelector) {
-    super(preemptionContext, isReservedPreemptionCandidatesSelector);
+      boolean isReservedPreemptionCandidatesSelector,
+      boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
+    super(preemptionContext, isReservedPreemptionCandidatesSelector,
+        allowQueuesBalanceAfterAllQueuesSatisfied);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
index 4d8afaf..3c97364 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
@@ -34,6 +34,7 @@ import java.util.Set;
 public abstract class PreemptionCandidatesSelector {
   protected CapacitySchedulerPreemptionContext preemptionContext;
   protected ResourceCalculator rc;
+  private long maximumKillWaitTime = -1;
 
   PreemptionCandidatesSelector(
       CapacitySchedulerPreemptionContext preemptionContext) {
@@ -77,4 +78,14 @@ public abstract class PreemptionCandidatesSelector {
     });
   }
 
+  public long getMaximumKillWaitTimeMs() {
+    if (maximumKillWaitTime > 0) {
+      return maximumKillWaitTime;
+    }
+    return preemptionContext.getDefaultMaximumKillWaitTimeout();
+  }
+
+  public void setMaximumKillWaitTime(long maximumKillWaitTime) {
+    this.maximumKillWaitTime = maximumKillWaitTime;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index cc69fba..036fd2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -131,6 +131,8 @@ public class ProportionalCapacityPreemptionPolicy
   private List<PreemptionCandidatesSelector> candidatesSelectionPolicies;
   private Set<String> allPartitions;
   private Set<String> leafQueueNames;
+  Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+      Set<RMContainer>>> pcsMap;
 
   // Preemptable Entities, synced from scheduler at every run
   private Map<String, PreemptableQueue> preemptableQueues;
@@ -249,7 +251,21 @@ public class ProportionalCapacityPreemptionPolicy
 
     // initialize candidates preemption selection policies
     candidatesSelectionPolicies.add(new FifoCandidatesSelector(this,
-        additionalPreemptionBasedOnReservedResource));
+        additionalPreemptionBasedOnReservedResource, false));
+
+    // Do we need to do preemption to balance queue even after queues get satisfied?
+    boolean isPreemptionToBalanceRequired = config.getBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED);
+    long maximumKillWaitTimeForPreemptionToQueueBalance = config.getLong(
+        CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+        CapacitySchedulerConfiguration.DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION);
+    if (isPreemptionToBalanceRequired) {
+      PreemptionCandidatesSelector selector = new FifoCandidatesSelector(this,
+          false, true);
+      selector.setMaximumKillWaitTime(maximumKillWaitTimeForPreemptionToQueueBalance);
+      candidatesSelectionPolicies.add(selector);
+    }
 
     // Do we need to specially consider intra queue
     boolean isIntraQueuePreemptionEnabled = config.getBoolean(
@@ -282,7 +298,8 @@ public class ProportionalCapacityPreemptionPolicy
         "select_based_on_reserved_containers = " +
           selectCandidatesForResevedContainers + "\n" +
         "additional_res_balance_based_on_reserved_containers = " +
-          additionalPreemptionBasedOnReservedResource);
+          additionalPreemptionBasedOnReservedResource + "\n" +
+        "Preemption-to-balance-queue-enabled = " + isPreemptionToBalanceRequired);
 
     csConfig = config;
   }
@@ -308,44 +325,60 @@ public class ProportionalCapacityPreemptionPolicy
   }
 
   private void preemptOrkillSelectedContainerAfterWait(
-      Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
-      long currentTime) {
+      Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+          Set<RMContainer>>> toPreemptPerSelector, long currentTime) {
+    int toPreemptCount = 0;
+    for (Map<ApplicationAttemptId, Set<RMContainer>> containers :
+        toPreemptPerSelector.values()) {
+      toPreemptCount += containers.size();
+    }
     if (LOG.isDebugEnabled()) {
       LOG.debug(
           "Starting to preempt containers for selectedCandidates and size:"
-              + selectedCandidates.size());
+              + toPreemptCount);
     }
 
     // preempt (or kill) the selected containers
-    for (Map.Entry<ApplicationAttemptId, Set<RMContainer>> e : selectedCandidates
+    // We need toPreemptPerSelector here to match list of containers to
+    // its selector so that we can get custom timeout per selector when
+    // checking if current container should be killed or not
+    for (Map.Entry<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+        Set<RMContainer>>> pc : toPreemptPerSelector
         .entrySet()) {
-      ApplicationAttemptId appAttemptId = e.getKey();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Send to scheduler: in app=" + appAttemptId
-            + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
-      }
-      for (RMContainer container : e.getValue()) {
-        // if we tried to preempt this for more than maxWaitTime
-        if (preemptionCandidates.get(container) != null
-            && preemptionCandidates.get(container)
-                + maxWaitTime <= currentTime) {
-          // kill it
-          rmContext.getDispatcher().getEventHandler().handle(
-              new ContainerPreemptEvent(appAttemptId, container,
-                  SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
-          preemptionCandidates.remove(container);
-        } else {
-          if (preemptionCandidates.get(container) != null) {
-            // We already updated the information to scheduler earlier, we need
-            // not have to raise another event.
-            continue;
+      Map<ApplicationAttemptId, Set<RMContainer>> cMap = pc.getValue();
+      if (cMap.size() > 0) {
+        for (Map.Entry<ApplicationAttemptId,
+            Set<RMContainer>> e : cMap.entrySet()) {
+          ApplicationAttemptId appAttemptId = e.getKey();
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Send to scheduler: in app=" + appAttemptId
+                + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
+          }
+          for (RMContainer container : e.getValue()) {
+            // if we tried to preempt this for more than maxWaitTime, this
+            // should be based on custom timeout per container per selector
+            if (preemptionCandidates.get(container) != null
+                && preemptionCandidates.get(container)
+                + pc.getKey().getMaximumKillWaitTimeMs() <= currentTime) {
+              // kill it
+              rmContext.getDispatcher().getEventHandler().handle(
+                  new ContainerPreemptEvent(appAttemptId, container,
+                      SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
+              preemptionCandidates.remove(container);
+            } else {
+              if (preemptionCandidates.get(container) != null) {
+                // We already updated the information to scheduler earlier, we need
+                // not have to raise another event.
+                continue;
+              }
+
+              //otherwise just send preemption events
+              rmContext.getDispatcher().getEventHandler().handle(
+                  new ContainerPreemptEvent(appAttemptId, container,
+                      SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
+              preemptionCandidates.put(container, currentTime);
+            }
           }
-
-          //otherwise just send preemption events
-          rmContext.getDispatcher().getEventHandler().handle(
-              new ContainerPreemptEvent(appAttemptId, container,
-                  SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
-          preemptionCandidates.put(container, currentTime);
         }
       }
     }
@@ -438,6 +471,8 @@ public class ProportionalCapacityPreemptionPolicy
     // queue and each application
     Map<ApplicationAttemptId, Set<RMContainer>> toPreempt =
         new HashMap<>();
+    Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+        Set<RMContainer>>> toPreemptPerSelector =  new HashMap<>();;
     for (PreemptionCandidatesSelector selector :
         candidatesSelectionPolicies) {
       long startTime = 0;
@@ -447,20 +482,27 @@ public class ProportionalCapacityPreemptionPolicy
                 selector.getClass().getName()));
         startTime = clock.getTime();
       }
-      toPreempt = selector.selectCandidates(toPreempt,
-          clusterResources, totalPreemptionAllowed);
+      Map<ApplicationAttemptId, Set<RMContainer>> curCandidates =
+          selector.selectCandidates(toPreempt, clusterResources,
+              totalPreemptionAllowed);
+      toPreemptPerSelector.putIfAbsent(selector, curCandidates);
 
       if (LOG.isDebugEnabled()) {
         LOG.debug(MessageFormat
             .format("{0} uses {1} millisecond to run",
                 selector.getClass().getName(), clock.getTime() - startTime));
         int totalSelected = 0;
+        int curSelected = 0;
         for (Set<RMContainer> set : toPreempt.values()) {
           totalSelected += set.size();
         }
+        for (Set<RMContainer> set : curCandidates.values()) {
+          curSelected += set.size();
+        }
         LOG.debug(MessageFormat
-            .format("So far, total {0} containers selected to be preempted",
-                totalSelected));
+            .format("So far, total {0} containers selected to be preempted, {1}"
+                    + " containers selected this round\n",
+                totalSelected, curSelected));
       }
     }
 
@@ -483,8 +525,10 @@ public class ProportionalCapacityPreemptionPolicy
 
     long currentTime = clock.getTime();
 
+    pcsMap = toPreemptPerSelector;
+
     // preempt (or kill) the selected containers
-    preemptOrkillSelectedContainerAfterWait(toPreempt, currentTime);
+    preemptOrkillSelectedContainerAfterWait(toPreemptPerSelector, currentTime);
 
     // cleanup staled preemption candidates
     cleanupStaledPreemptionCandidates(currentTime);
@@ -689,6 +733,12 @@ public class ProportionalCapacityPreemptionPolicy
     return queueToPartitions;
   }
 
+  @VisibleForTesting
+  Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+      Set<RMContainer>>> getToPreemptCandidatesPerSelector() {
+    return pcsMap;
+  }
+
   @Override
   public int getClusterMaxApplicationPriority() {
     return scheduler.getMaxClusterLevelAppPriority().getPriority();
@@ -730,4 +780,9 @@ public class ProportionalCapacityPreemptionPolicy
   public IntraQueuePreemptionOrderPolicy getIntraQueuePreemptionOrderPolicy() {
     return intraQueuePreemptionOrderPolicy;
   }
+
+  @Override
+  public long getDefaultMaximumKillWaitTimeout() {
+    return maxWaitTime;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
index 4a169af..78a9988 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
@@ -380,6 +380,7 @@ public class QueuePriorityContainerCandidateSelector
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
       Resource clusterResource,
       Resource totalPreemptedResourceAllowed) {
+    Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
     // Initialize digraph from queues
     // TODO (wangda): only do this when queue refreshed.
     priorityDigraph.clear();
@@ -388,7 +389,7 @@ public class QueuePriorityContainerCandidateSelector
     // When all queues are set to same priority, or priority is not respected,
     // direct return.
     if (priorityDigraph.isEmpty()) {
-      return selectedCandidates;
+      return curCandidates;
     }
 
     // Save parameters to be shared by other methods
@@ -478,13 +479,9 @@ public class QueuePriorityContainerCandidateSelector
                 .getReservedResource());
           }
 
-          Set<RMContainer> containers = selectedCandidates.get(
-              c.getApplicationAttemptId());
-          if (null == containers) {
-            containers = new HashSet<>();
-            selectedCandidates.put(c.getApplicationAttemptId(), containers);
-          }
-          containers.add(c);
+          // Add to preemptMap
+          CapacitySchedulerPreemptionUtils.addToPreemptMap(selectedCandidates,
+              curCandidates, c.getApplicationAttemptId(), c);
 
           // Update totalPreemptionResourceAllowed
           Resources.subtractFrom(totalPreemptedResourceAllowed,
@@ -504,7 +501,6 @@ public class QueuePriorityContainerCandidateSelector
         }
       }
     }
-
-    return selectedCandidates;
+    return curCandidates;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
index ff100d9..bdb7e8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
@@ -31,7 +31,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -63,7 +62,7 @@ public class ReservedContainerCandidatesSelector
       CapacitySchedulerPreemptionContext preemptionContext) {
     super(preemptionContext);
     preemptableAmountCalculator = new PreemptableResourceCalculator(
-        preemptionContext, true);
+        preemptionContext, true, false);
   }
 
   @Override
@@ -71,6 +70,7 @@ public class ReservedContainerCandidatesSelector
       Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
       Resource clusterResource,
       Resource totalPreemptedResourceAllowed) {
+    Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
     // Calculate how much resources we need to preempt
     preemptableAmountCalculator.computeIdealAllocation(clusterResource,
         totalPreemptedResourceAllowed);
@@ -101,14 +101,10 @@ public class ReservedContainerCandidatesSelector
           selectedCandidates, totalPreemptedResourceAllowed, false);
       if (null != preemptionResult) {
         for (RMContainer c : preemptionResult.selectedContainers) {
-          ApplicationAttemptId appId = c.getApplicationAttemptId();
-          Set<RMContainer> containers = selectedCandidates.get(appId);
-          if (null == containers) {
-            containers = new HashSet<>();
-            selectedCandidates.put(appId, containers);
-          }
+          // Add to preemptMap
+          CapacitySchedulerPreemptionUtils.addToPreemptMap(selectedCandidates,
+              curCandidates, c.getApplicationAttemptId(), c);
 
-          containers.add(c);
           if (LOG.isDebugEnabled()) {
             LOG.debug(this.getClass().getName() + " Marked container=" + c
                 .getContainerId() + " from queue=" + c.getQueueName()
@@ -118,7 +114,7 @@ public class ReservedContainerCandidatesSelector
       }
     }
 
-    return selectedCandidates;
+    return curCandidates;
   }
 
   private Resource getPreemptableResource(String queueName,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 4214acc..4fb1862 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -138,7 +138,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
   // This function "accepts" all the resources it can (pending) and return
   // the unused ones
   Resource offer(Resource avail, ResourceCalculator rc,
-      Resource clusterResource, boolean considersReservedResource) {
+      Resource clusterResource, boolean considersReservedResource,
+      boolean allowQueueBalanceAfterAllSafisfied) {
     Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
         Resources.subtract(getMax(), idealAssigned),
         Resource.newInstance(0, 0));
@@ -179,7 +180,10 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
     // leaf queues. Such under-utilized leaf queue could preemption resources
     // from over-utilized leaf queue located at other hierarchies.
 
-    accepted = filterByMaxDeductAssigned(rc, clusterResource, accepted);
+    // Allow queues can continue grow and balance even if all queues are satisfied.
+    if (!allowQueueBalanceAfterAllSafisfied) {
+      accepted = filterByMaxDeductAssigned(rc, clusterResource, accepted);
+    }
 
     // accepted so far contains the "quota acceptable" amount, we now filter by
     // locality acceptable

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 76eaac0..f94654e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1460,6 +1460,23 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
   public static final String DEFAULT_INTRAQUEUE_PREEMPTION_ORDER_POLICY = "userlimit_first";
 
   /**
+   * Should we allow queues continue grow after all queue reaches their
+   * guaranteed capacity.
+   */
+  public static final String PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED =
+      PREEMPTION_CONFIG_PREFIX + "preemption-to-balance-queue-after-satisfied.enabled";
+  public static final boolean DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED = false;
+
+  /**
+   * How long we will wait to balance queues, by default it is 5 mins.
+   */
+  public static final String MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION =
+      PREEMPTION_CONFIG_PREFIX + "preemption-to-balance-queue-after-satisfied.max-wait-before-kill";
+  public static final long
+      DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION =
+      300 * 1000;
+
+  /**
    * Maximum application for a queue to be used when application per queue is
    * not defined.To be consistent with previous version the default value is set
    * as UNDEFINED.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
index 6a953cf..38c2a2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
 import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
 import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
 import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -538,4 +539,61 @@ public class TestPreemptionForQueueWithPriorities
         new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
             getAppAttemptId(1))));
   }
+
+  @Test
+  public void testPriorityPreemptionForBalanceBetweenSatisfiedQueues()
+      throws IOException {
+    /**
+     * All queues are beyond guarantee, c has higher priority than b.
+     * c ask for more resource, and there is no idle left, c should preempt
+     * some resource from b but won’t let b under its guarantee.
+     *
+     * Queue structure is:
+     *
+     * <pre>
+     *        root
+     *       / |  \
+     *      a  b   c
+     * </pre>
+     *
+     * For priorities
+     * - a=1
+     * - b=1
+     * - c=2
+     *
+     */
+    String labelsConfig = "=100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100 100 100 100]);" + //root
+            "-a(=[30 100 0 0]){priority=1};" + // a
+            "-b(=[30 100 40 50]){priority=1};" + // b
+            "-c(=[40 100 60 25]){priority=2}";   // c
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "b\t(1,1,n1,,40,false);" + // app1 in b
+            "c\t(1,1,n1,,60,false)"; // app2 in c
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+    CapacitySchedulerConfiguration newConf =
+        new CapacitySchedulerConfiguration(conf);
+    boolean isPreemptionToBalanceRequired = true;
+    newConf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        isPreemptionToBalanceRequired);
+    when(cs.getConfiguration()).thenReturn(newConf);
+    policy.editSchedule();
+
+    // IdealAssigned b: 30 c: 70. initIdealAssigned: b: 30 c: 40, even though
+    // b and c has same relativeAssigned=1.0f(idealAssigned / guaranteed),
+    // since c has higher priority, c will be put in mostUnderServedQueue and
+    // get all remain 30 capacity.
+    verify(mDisp, times(10)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(1))));
+    verify(mDisp, never()).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(2))));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
new file mode 100644
index 0000000..22e8f63
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TestProportionalCapacityPreemptionPolicyPreemptToBalance
+    extends ProportionalCapacityPreemptionPolicyMockFramework {
+
+  @Test
+  public void testPreemptionToBalanceDisabled() throws IOException {
+    String labelsConfig = "=100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100 100 100 100]);" + //root
+            "-a(=[30 100 10 30]);" + // a
+            "-b(=[30 100 40 30]);" + // b
+            "-c(=[30 100 50 30]);" + // c
+            "-d(=[10 100 0 0])";   // d
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a\t(1,1,n1,,10,false);" + // app1 in a
+            "b\t(1,1,n1,,40,false);" + // app2 in b
+            "c\t(1,1,n1,,50,false)"; // app3 in c
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+    policy.editSchedule();
+
+    // I_A: A:30 B:35 C:35, preempt 5 from B and 15 from C to A
+    verify(mDisp, times(5)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(2))));
+    verify(mDisp, times(15)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(3))));
+
+    assertEquals(30, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(35, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(35, policy.getQueuePartitions().get("c")
+        .get("").getIdealAssigned().getMemorySize());
+  }
+
+  @Test
+  public void testPreemptionToBalanceEnabled() throws IOException {
+    String labelsConfig = "=100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100 100 100 100]);" + //root
+            "-a(=[30 100 10 30]);" + // a
+            "-b(=[30 100 40 30]);" + // b
+            "-c(=[30 100 50 30]);" + // c
+            "-d(=[10 100 0 0])";   // d
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a\t(1,1,n1,,10,false);" + // app1 in a
+            "b\t(1,1,n1,,40,false);" + // app2 in b
+            "c\t(1,1,n1,,50,false)"; // app3 in c
+
+    // enable preempt to balance and ideal assignment will change.
+    boolean isPreemptionToBalanceEnabled = true;
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        isPreemptionToBalanceEnabled);
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+    policy.editSchedule();
+
+    // I_A: A:33 B:33 C:33, preempt 7 from B and 17 from C to A
+    verify(mDisp, times(7)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(2))));
+    verify(mDisp, times(17)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(3))));
+
+    assertEquals(33, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(33, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(33, policy.getQueuePartitions().get("c")
+        .get("").getIdealAssigned().getMemorySize());
+  }
+
+
+  @Test
+  public void testPreemptionToBalanceUsedPlusPendingLessThanGuaranteed()
+      throws IOException{
+    String labelsConfig = "=100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100 100 100 100]);" + //root
+            "-a(=[30 100 10 6]);" + // a
+            "-b(=[30 100 40 30]);" + // b
+            "-c(=[30 100 50 30]);" + // c
+            "-d(=[10 100 0 0])";   // d
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a\t(1,1,n1,,10,false);" + // app1 in a
+            "b\t(1,1,n1,,40,false);" + // app2 in b
+            "c\t(1,1,n1,,50,false)"; // app3 in c
+
+    boolean isPreemptionToBalanceEnabled = true;
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        isPreemptionToBalanceEnabled);
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+    policy.editSchedule();
+
+    // I_A: A:15 B:42 C:43, preempt 7 from B and 17 from C to A
+    verify(mDisp, times(8)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+            getAppAttemptId(3))));
+
+    assertEquals(16, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(42, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(42, policy.getQueuePartitions().get("c")
+        .get("").getIdealAssigned().getMemorySize());
+  }
+
+  @Test
+  public void testPreemptionToBalanceWithVcoreResource() throws IOException {
+    Logger.getRootLogger().setLevel(Level.DEBUG);
+    String labelsConfig = "=100:100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100:100 100:100 100:100 120:140]);" + //root
+            "-a(=[60:60 100:100 40:40 70:40]);" + // a
+            "-b(=[40:40 100:100 60:60 50:100])";   // b
+
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a\t(1,1:1,n1,,40,false);" + // app1 in a
+            "b\t(1,1:1,n1,,60,false)"; // app2 in b
+
+    boolean isPreemptionToBalanceEnabled = true;
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        isPreemptionToBalanceEnabled);
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true);
+    policy.editSchedule();
+
+    // 21 containers will be preempted here
+    verify(mDisp, times(21)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.
+            IsPreemptionRequestFor(getAppAttemptId(2))));
+
+    assertEquals(60, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(60, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getVirtualCores());
+    assertEquals(40, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(40, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getVirtualCores());
+  }
+
+  @Test
+  public void testPreemptionToBalanceWithConfiguredTimeout() throws IOException {
+    Logger.getRootLogger().setLevel(Level.DEBUG);
+    String labelsConfig = "=100:100,true"; // default partition
+    String nodesConfig = "n1="; // only one node
+    String queuesConfig =
+        // guaranteed,max,used,pending
+        "root(=[100:100 100:100 100:100 120:140]);" + //root
+            "-a(=[60:60 100:100 40:40 70:40]);" + // a
+            "-b(=[40:40 100:100 60:60 50:100])";   // b
+
+    String appsConfig =
+        //queueName\t(priority,resource,host,expression,#repeat,reserved)
+        "a\t(1,1:1,n1,,40,false);" + // app1 in a
+            "b\t(1,1:1,n1,,60,false)"; // app2 in b
+
+    boolean isPreemptionToBalanceEnabled = true;
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        isPreemptionToBalanceEnabled);
+    final long FB_MAX_BEFORE_KILL = 60 *1000;
+    conf.setLong(
+        CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+        FB_MAX_BEFORE_KILL);
+
+    buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true);
+    policy.editSchedule();
+
+    Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+        Set<RMContainer>>> pcps= policy.getToPreemptCandidatesPerSelector();
+
+    String FIFO_CANDIDATE_SELECTOR = "FifoCandidatesSelector";
+    boolean hasFifoSelector = false;
+    for (Map.Entry<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+        Set<RMContainer>>> pc : pcps.entrySet()) {
+      if (pc.getKey().getClass().getSimpleName().equals(FIFO_CANDIDATE_SELECTOR)) {
+        FifoCandidatesSelector pcs = (FifoCandidatesSelector) pc.getKey();
+        if (pcs.getAllowQueuesBalanceAfterAllQueuesSatisfied() == true) {
+          hasFifoSelector = true;
+          assertEquals(pcs.getMaximumKillWaitTimeMs(), FB_MAX_BEFORE_KILL);
+        }
+      }
+    }
+
+    assertEquals(hasFifoSelector, true);
+
+    // 21 containers will be preempted here
+    verify(mDisp, times(21)).handle(argThat(
+        new TestProportionalCapacityPreemptionPolicy.
+            IsPreemptionRequestFor(getAppAttemptId(2))));
+
+    assertEquals(60, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(60, policy.getQueuePartitions().get("a")
+        .get("").getIdealAssigned().getVirtualCores());
+    assertEquals(40, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getMemorySize());
+    assertEquals(40, policy.getQueuePartitions().get("b")
+        .get("").getIdealAssigned().getVirtualCores());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 2aff82d..800789a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -1111,5 +1111,116 @@ public class TestCapacitySchedulerSurgicalPreemption
     rm1.close();
   }
 
+  @Test(timeout = 600000)
+  public void testPreemptionToBalanceWithCustomTimeout() throws Exception {
+    /**
+     * Test case: Submit two application (app1/app2) to different queues, queue
+     * structure:
+     *
+     * <pre>
+     *             Root
+     *            /  |  \
+     *           a   b   c
+     *          10   20  70
+     * </pre>
+     *
+     * 1) Two nodes (n1/n2) in the cluster, each of them has 20G.
+     *
+     * 2) app1 submit to queue-b, asks for 1G * 5
+     *
+     * 3) app2 submit to queue-c, ask for one 4G container (for AM)
+     *
+     * After preemption, we should expect:
+     * 1. Preempt 4 containers from app1
+     * 2. the selected containers will be killed after configured timeout.
+     * 3. AM of app2 successfully allocated.
+     */
+    conf.setBoolean(
+        CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+        true);
+    conf.setLong(
+        CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+        20*1000);
+    CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
+        this.conf);
+
+    MockRM rm1 = new MockRM(conf);
+    rm1.getRMContext().setNodeLabelManager(mgr);
+    rm1.start();
+
+    MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB);
+    MockNM nm2 = rm1.registerNode("h2:1234", 20 * GB);
+    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+    RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+    RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+    // launch an app to queue, AM container should be launched in nm1
+    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+    am1.allocate("*", 1 * GB, 38, new ArrayList<ContainerId>());
+
+    // Do allocation for node1/node2
+    for (int i = 0; i < 38; i++) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+    }
+
+    // App1 should have 39 containers now
+    FiCaSchedulerApp schedulerApp1 = cs.getApplicationAttempt(
+        am1.getApplicationAttemptId());
+    Assert.assertEquals(39, schedulerApp1.getLiveContainers().size());
+    // 20 from n1 and 19 from n2
+    waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode1.getNodeID()),
+        am1.getApplicationAttemptId(), 20);
+    waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode2.getNodeID()),
+        am1.getApplicationAttemptId(), 19);
+
+
+    // Submit app2 to queue-c and asks for a 4G container for AM
+    RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c");
+    FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt(
+        ApplicationAttemptId.newInstance(app2.getApplicationId(), 1));
+
+    // Call editSchedule: containers are selected to be preemption candidate
+    SchedulingMonitorManager smm = ((CapacityScheduler) rm1.
+        getResourceScheduler()).getSchedulingMonitorManager();
+    SchedulingMonitor smon = smm.getAvailableSchedulingMonitor();
+    ProportionalCapacityPreemptionPolicy editPolicy =
+        (ProportionalCapacityPreemptionPolicy) smon.getSchedulingEditPolicy();
+    editPolicy.editSchedule();
+    Assert.assertEquals(4, editPolicy.getToPreemptContainers().size());
+
+    // check live containers immediately, nothing happen
+    Assert.assertEquals(39, schedulerApp1.getLiveContainers().size());
+
+    Thread.sleep(20*1000);
+    // Call editSchedule again: selected containers are killed
+    editPolicy.editSchedule();
+    waitNumberOfLiveContainersFromApp(schedulerApp1, 35);
+
+    // Call allocation, containers are reserved
+    cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+    cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+    waitNumberOfReservedContainersFromApp(schedulerApp2, 1);
+
+    // Call editSchedule twice and allocation once, container should get allocated
+    editPolicy.editSchedule();
+    editPolicy.editSchedule();
+
+    int tick = 0;
+    while (schedulerApp2.getLiveContainers().size() != 1 && tick < 10) {
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+      cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+      tick++;
+      Thread.sleep(100);
+    }
+    waitNumberOfReservedContainersFromApp(schedulerApp2, 0);
+
+    rm1.close();
+
+
+  }
+
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[48/50] [abbrv] hadoop git commit: YARN-8481. AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters. Contributed by Botong Huang.

Posted by gi...@apache.org.
YARN-8481. AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters. Contributed by Botong Huang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdb08442
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdb08442
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdb08442

Branch: refs/heads/HADOOP-15461
Commit: cdb084426bc27a9f902da0b6927a3354a307dd82
Parents: d36f6b9
Author: Giovanni Matteo Fumarola <gi...@apache.com>
Authored: Fri Jun 29 11:47:30 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 11:47:30 2018 -0700

----------------------------------------------------------------------
 .../policies/amrmproxy/BroadcastAMRMProxyPolicy.java     | 11 -----------
 .../policies/amrmproxy/RejectAMRMProxyPolicy.java        |  4 ----
 .../TestBroadcastAMRMProxyFederationPolicy.java          | 11 +++--------
 3 files changed, 3 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
index 679f4d5..7fddb8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
@@ -19,10 +19,8 @@
 package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -30,7 +28,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
 import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
 import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.UnknownSubclusterException;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
 import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
 
@@ -40,8 +37,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
  */
 public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
 
-  private Set<SubClusterId> knownClusterIds = new HashSet<>();
-
   @Override
   public void reinitialize(
       FederationPolicyInitializationContext policyContext)
@@ -65,7 +60,6 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
     // simply broadcast the resource request to all sub-clusters
     for (SubClusterId subClusterId : activeSubclusters.keySet()) {
       answer.put(subClusterId, resourceRequests);
-      knownClusterIds.add(subClusterId);
     }
 
     return answer;
@@ -74,11 +68,6 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
   @Override
   public void notifyOfResponse(SubClusterId subClusterId,
       AllocateResponse response) throws YarnException {
-    if (!knownClusterIds.contains(subClusterId)) {
-      throw new UnknownSubclusterException(
-          "The response is received from a subcluster that is unknown to this "
-              + "policy.");
-    }
     // stateless policy does not care about responses
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
index 3783df6..4500606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -18,10 +18,8 @@
 
 package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
 
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -38,8 +36,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
  */
 public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
 
-  private Set<SubClusterId> knownClusterIds = new HashSet<>();
-
   @Override
   public void reinitialize(FederationPolicyInitializationContext policyContext)
       throws FederationPolicyInitializationException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
index a21f53d..df5da85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -89,7 +89,7 @@ public class TestBroadcastAMRMProxyFederationPolicy
   }
 
   @Test
-  public void testNotifyOfResponse() throws Exception {
+  public void testNotifyOfResponseFromUnknownSubCluster() throws Exception {
     String[] hosts = new String[] {"host1", "host2" };
     List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
         .createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
@@ -97,13 +97,8 @@ public class TestBroadcastAMRMProxyFederationPolicy
         ((FederationAMRMProxyPolicy) getPolicy())
             .splitResourceRequests(resourceRequests);
 
-    try {
-      ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
-          SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
-      Assert.fail();
-    } catch (FederationPolicyException f) {
-      System.out.println("Expected: " + f.getMessage());
-    }
+    ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+        SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
 
     ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
         SubClusterId.newInstance("sc1"), mock(AllocateResponse.class));


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[18/50] [abbrv] hadoop git commit: HDDS-192:Create new SCMCommand to request a replication of a container. Contributed by Elek Marton

Posted by gi...@apache.org.
HDDS-192:Create new SCMCommand to request a replication of a container. Contributed by Elek Marton


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/238fe00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/238fe00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/238fe00a

Branch: refs/heads/HADOOP-15461
Commit: 238fe00ad2692154f6a382f35735169ee5e4af2c
Parents: 35ec940
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Jun 25 21:12:05 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Jun 25 21:12:05 2018 -0700

----------------------------------------------------------------------
 .../statemachine/DatanodeStateMachine.java      |  3 +
 .../ReplicateContainerCommandHandler.java       | 67 ++++++++++++++
 .../states/endpoint/HeartbeatEndpointTask.java  | 12 +++
 .../commands/ReplicateContainerCommand.java     | 94 ++++++++++++++++++++
 .../StorageContainerDatanodeProtocol.proto      | 12 ++-
 .../scm/server/SCMDatanodeProtocolServer.java   | 11 +++
 .../TestReplicateContainerHandler.java          | 71 +++++++++++++++
 7 files changed, 269 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index dc4e673..b073d7b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
     .CommandDispatcher;
 import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
     .DeleteBlocksCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
+    .ReplicateContainerCommandHandler;
 import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.util.Time;
@@ -95,6 +97,7 @@ public class DatanodeStateMachine implements Closeable {
         .addHandler(new CloseContainerCommandHandler())
         .addHandler(new DeleteBlocksCommandHandler(
             container.getContainerManager(), conf))
+        .addHandler(new ReplicateContainerCommandHandler())
         .setConnectionManager(connectionManager)
         .setContainer(container)
         .setContext(context)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
new file mode 100644
index 0000000..b4e83b7
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine
+    .SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Command handler to copy containers from sources.
+ */
+public class ReplicateContainerCommandHandler implements CommandHandler {
+  static final Logger LOG =
+      LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
+
+  private int invocationCount;
+
+  private long totalTime;
+
+  @Override
+  public void handle(SCMCommand command, OzoneContainer container,
+      StateContext context, SCMConnectionManager connectionManager) {
+    LOG.warn("Replicate command is not yet handled");
+
+  }
+
+  @Override
+  public SCMCommandProto.Type getCommandType() {
+    return Type.replicateContainerCommand;
+  }
+
+  @Override
+  public int getInvocationCount() {
+    return this.invocationCount;
+  }
+
+  @Override
+  public long getAverageRunTime() {
+    if (invocationCount > 0) {
+      return totalTime / invocationCount;
+    }
+    return 0;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 1ee6375..260a245 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.ozone.container.common.statemachine
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
 import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
 import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -196,6 +198,16 @@ public class HeartbeatEndpointTask
         }
         this.context.addCommand(closeContainer);
         break;
+      case replicateContainerCommand:
+        ReplicateContainerCommand replicateContainerCommand =
+            ReplicateContainerCommand.getFromProtobuf(
+                commandResponseProto.getReplicateContainerCommandProto());
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Received SCM container replicate request for container {}",
+              replicateContainerCommand.getContainerID());
+        }
+        this.context.addCommand(replicateContainerCommand);
+        break;
       default:
         throw new IllegalArgumentException("Unknown response : "
             + commandResponseProto.getCommandType().name());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
new file mode 100644
index 0000000..834318b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto
+    .Builder;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * SCM command to request replication of a container.
+ */
+public class ReplicateContainerCommand
+    extends SCMCommand<ReplicateContainerCommandProto> {
+
+  private final long containerID;
+
+  private final List<DatanodeDetails> sourceDatanodes;
+
+  public ReplicateContainerCommand(long containerID,
+      List<DatanodeDetails> sourceDatanodes) {
+    this.containerID = containerID;
+    this.sourceDatanodes = sourceDatanodes;
+  }
+
+  @Override
+  public Type getType() {
+    return SCMCommandProto.Type.replicateContainerCommand;
+  }
+
+  @Override
+  public byte[] getProtoBufMessage() {
+    return getProto().toByteArray();
+  }
+
+  public ReplicateContainerCommandProto getProto() {
+    Builder builder = ReplicateContainerCommandProto.newBuilder()
+        .setContainerID(containerID);
+    for (DatanodeDetails dd : sourceDatanodes) {
+      builder.addSources(dd.getProtoBufMessage());
+    }
+    return builder.build();
+  }
+
+  public static ReplicateContainerCommand getFromProtobuf(
+      ReplicateContainerCommandProto protoMessage) {
+    Preconditions.checkNotNull(protoMessage);
+
+    List<DatanodeDetails> datanodeDetails =
+        protoMessage.getSourcesList()
+        .stream()
+        .map(DatanodeDetails::getFromProtoBuf)
+        .collect(Collectors.toList());
+
+    return new ReplicateContainerCommand(protoMessage.getContainerID(),
+        datanodeDetails);
+
+  }
+
+  public long getContainerID() {
+    return containerID;
+  }
+
+  public List<DatanodeDetails> getSourceDatanodes() {
+    return sourceDatanodes;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index f6aba05..54230c1 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -172,6 +172,7 @@ message SCMCommandProto {
     deleteBlocksCommand = 2;
     closeContainerCommand = 3;
     deleteContainerCommand = 4;
+    replicateContainerCommand = 5;
   }
   // TODO: once we start using protoc 3.x, refactor this message using "oneof"
   required Type commandType = 1;
@@ -179,6 +180,7 @@ message SCMCommandProto {
   optional DeleteBlocksCommandProto deleteBlocksCommandProto = 3;
   optional CloseContainerCommandProto closeContainerCommandProto = 4;
   optional DeleteContainerCommandProto deleteContainerCommandProto = 5;
+  optional ReplicateContainerCommandProto replicateContainerCommandProto = 6;
 }
 
 /**
@@ -227,13 +229,21 @@ message CloseContainerCommandProto {
 }
 
 /**
-This command asks the datanode to close a specific container.
+This command asks the datanode to delete a specific container.
 */
 message DeleteContainerCommandProto {
   required int64 containerID = 1;
 }
 
 /**
+This command asks the datanode to replicate a container from specific sources.
+*/
+message ReplicateContainerCommandProto {
+  required int64 containerID = 1;
+  repeated DatanodeDetailsProto sources = 2;
+}
+
+/**
  * Protocol used from a datanode to StorageContainerManager.
  *
  * Please see the request and response messages for details of the RPC calls.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 7d16161..eb5ce1a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
 import org.apache.hadoop.hdds.protocol.proto
@@ -63,6 +64,9 @@ import static org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMCommandProto
     .Type.deleteBlocksCommand;
 import static org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type
+    .replicateContainerCommand;
+import static org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMCommandProto
     .Type.reregisterCommand;
 
@@ -77,6 +81,7 @@ import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
 import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
 import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
 import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
 import org.apache.hadoop.ozone.protocolPB
@@ -293,6 +298,12 @@ public class SCMDatanodeProtocolServer implements
           .setCloseContainerCommandProto(
               ((CloseContainerCommand) cmd).getProto())
           .build();
+    case replicateContainerCommand:
+      return builder
+          .setCommandType(replicateContainerCommand)
+          .setReplicateContainerCommandProto(
+              ((ReplicateContainerCommand)cmd).getProto())
+          .build();
     default:
       throw new IllegalArgumentException("Not implemented");
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
new file mode 100644
index 0000000..a5b101f
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_CONTAINER_SIZE_GB;
+import org.junit.Test;
+
+/**
+ * Tests the behavior of the datanode, when replicate container command is
+ * received.
+ */
+public class TestReplicateContainerHandler {
+
+  @Test
+  public void test() throws IOException, TimeoutException, InterruptedException,
+      OzoneException {
+
+    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+        .captureLogs(ReplicateContainerCommandHandler.LOG);
+
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(OZONE_SCM_CONTAINER_SIZE_GB, "1");
+    MiniOzoneCluster cluster =
+        MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+    cluster.waitForClusterToBeReady();
+
+    DatanodeDetails datanodeDetails =
+        cluster.getHddsDatanodes().get(0).getDatanodeDetails();
+    //send the order to replicate the container
+    cluster.getStorageContainerManager().getScmNodeManager()
+        .addDatanodeCommand(datanodeDetails.getUuid(),
+            new ReplicateContainerCommand(1L,
+                new ArrayList<>()));
+
+    //TODO: here we test only the serialization/unserialization as
+    // the implementation is not yet done
+    GenericTestUtils
+        .waitFor(() -> logCapturer.getOutput().contains("not yet handled"), 500,
+            5 * 1000);
+
+  }
+
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[10/50] [abbrv] hadoop git commit: YARN-8443. Total #VCores in cluster metrics is wrong when CapacityScheduler reserved some containers. Contributed by Tao Yang.

Posted by gi...@apache.org.
YARN-8443. Total #VCores in cluster metrics is wrong when CapacityScheduler reserved some containers. Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/440140ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/440140ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/440140ce

Branch: refs/heads/HADOOP-15461
Commit: 440140cea6718229094a3d2b97b9b9bd28b95d9b
Parents: e16e5b3
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Jun 25 09:15:31 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Jun 25 09:15:31 2018 +0800

----------------------------------------------------------------------
 .../yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/440140ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 84f70d9..69d88aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -101,7 +101,7 @@ public class ClusterMetricsInfo {
       CapacityScheduler cs = (CapacityScheduler) rs;
       this.totalMB = availableMB + allocatedMB + reservedMB;
       this.totalVirtualCores =
-          availableVirtualCores + allocatedVirtualCores + containersReserved;
+          availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
       // TODO, add support of other schedulers to get total used resources
       // across partition.
       if (cs.getRootQueue() != null


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[19/50] [abbrv] hadoop git commit: YARN-8214. Change default RegistryDNS port. Contributed by Billie Rinaldi

Posted by gi...@apache.org.
YARN-8214.  Change default RegistryDNS port.
            Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e586330
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e586330
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e586330

Branch: refs/heads/HADOOP-15461
Commit: 3e586330eb2c7db0b884fe328b171fb27ce545fa
Parents: 238fe00
Author: Eric Yang <ey...@apache.org>
Authored: Tue Jun 26 14:25:15 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Jun 26 14:25:15 2018 -0400

----------------------------------------------------------------------
 .../hadoop/registry/client/api/RegistryConstants.java     |  2 +-
 .../src/site/markdown/yarn-service/RegistryDNS.md         | 10 +++++-----
 .../src/site/markdown/yarn-service/ServiceDiscovery.md    |  6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index cfa2d65..bd97a5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -95,7 +95,7 @@ public interface RegistryConstants {
   /**
    * Default DNS port number.
    */
-  int DEFAULT_DNS_PORT = 5353;
+  int DEFAULT_DNS_PORT = 5335;
 
   /**
    * DNSSEC Enabled?

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
index 2307e5c..642d26e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -58,7 +58,7 @@ primary DNS server can be configured to forward a zone to the registry DNS
 server.
 2. The DNS Server exposes a port that can receive both TCP and UDP requests per
 DNS standards. The default port for DNS protocols is not in the restricted
-range (5353). However, existing DNS assets may only allow zone forwarding to
+range (5335). However, existing DNS assets may only allow zone forwarding to
 non-custom ports. To support this, the registry DNS server can be started in
 privileged mode.
 
@@ -136,7 +136,7 @@ standard DNS requests from users or other DNS servers (for example, DNS servers
 RegistryDNS service configured as a forwarder).
 
 ## Start the DNS Server
-By default, the DNS server runs on non-privileged port `5353`. Start the server
+By default, the DNS server runs on non-privileged port `5335`. Start the server
 with:
 ```
 yarn --daemon start registrydns
@@ -157,7 +157,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
 | hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
 | hadoop.registry.dns.domain-name  | The domain name for Hadoop cluster associated records.  |
 | hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind.  |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353.  |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5335.  |
 | hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false.  |
 | hadoop.registry.dns.public-key  | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests.  |
 | hadoop.registry.dns.private-key-file  | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation.  |
@@ -174,10 +174,10 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
   </property>
 
   <property>
-    <description>The port number for the DNS listener. The default port is 5353.
+    <description>The port number for the DNS listener. The default port is 5335.
     If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
     <name>hadoop.registry.dns.bind-port</name>
-    <value>5353</value>
+    <value>5335</value>
   </property>
 
   <property>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index 7ee16dd..6b93f3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -94,10 +94,10 @@ section of [Registry DNS](RegistryDNS.html).
   </property>
 
   <property>
-    <description>The port number for the DNS listener. The default port is 5353.
+    <description>The port number for the DNS listener. The default port is 5335.
     If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
     <name>hadoop.registry.dns.bind-port</name>
-    <value>5353</value>
+    <value>5335</value>
   </property>
 
   <property>
@@ -135,7 +135,7 @@ To configure Registry DNS to serve reverse lookup for `172.17.0.0/24`
   </property>
 ```
 ## Start Registry DNS Server
-By default, the DNS server runs on non-privileged port `5353`. Start the server
+By default, the DNS server runs on non-privileged port `5335`. Start the server
 with:
 ```
 yarn --daemon start registrydns


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[03/50] [abbrv] hadoop git commit: MAPREDUCE-7114. Make FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.

Posted by gi...@apache.org.
MAPREDUCE-7114. Make FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae055622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae055622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae055622

Branch: refs/heads/HADOOP-15461
Commit: ae055622edeb3cbf82baa6ed952fc2abc84c021e
Parents: 55fad6a
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Fri Jun 22 13:05:41 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Fri Jun 22 13:05:41 2018 -0700

----------------------------------------------------------------------
 .../hadoop/mapred/uploader/FrameworkUploader.java      |  2 +-
 .../hadoop/mapred/uploader/TestFrameworkUploader.java  | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index 5316f38..d2116c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -409,7 +409,7 @@ public class FrameworkUploader implements Runnable {
             linkPath == null ? null : linkPath.getParent();
         java.nio.file.Path normalizedLinkPath =
             linkPathParent == null ? null : linkPathParent.normalize();
-        if (normalizedLinkPath != null && jarParent.equals(
+        if (normalizedLinkPath != null && jarParent.normalize().equals(
             normalizedLinkPath)) {
           LOG.info(String.format("Ignoring same directory link %s to %s",
               jarPath.toString(), link.toString()));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
index c12902c..9c72f72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
@@ -440,6 +440,19 @@ public class TestFrameworkUploader {
       }
       Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
 
+      // Create a symlink to the target with /./ in the path
+      symlinkToTarget = new File(parent.getAbsolutePath() +
+            "/./symlinkToTarget2.txt");
+      try {
+        Files.createSymbolicLink(
+            Paths.get(symlinkToTarget.getAbsolutePath()),
+            Paths.get(targetFile.getAbsolutePath()));
+      } catch (UnsupportedOperationException e) {
+        // Symlinks are not supported, so ignore the test
+        Assume.assumeTrue(false);
+      }
+      Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
+
       // Create a symlink outside the current directory
       File symlinkOutside = new File(parent, "symlinkToParent.txt");
       try {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[02/50] [abbrv] hadoop git commit: HADOOP-15416. Clear error message in S3Guard diff if source not found. Contributed by Gabor Bota.

Posted by gi...@apache.org.
HADOOP-15416. Clear error message in S3Guard diff if source not found. Contributed by Gabor Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55fad6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55fad6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55fad6a3

Branch: refs/heads/HADOOP-15461
Commit: 55fad6a3de3125d9e7e2e9a5f8fa5b1b22a1de60
Parents: 6432128
Author: Sean Mackrory <ma...@apache.org>
Authored: Fri Jun 22 11:36:36 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Fri Jun 22 11:36:56 2018 -0600

----------------------------------------------------------------------
 .../main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fad6a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index fbffba9..ac10e08 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -805,7 +805,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
      */
     private void compareDir(FileStatus msDir, FileStatus s3Dir,
                             PrintStream out) throws IOException {
-      Preconditions.checkArgument(!(msDir == null && s3Dir == null));
+      Preconditions.checkArgument(!(msDir == null && s3Dir == null),
+          "The path does not exist in metadata store and on s3.");
+
       if (msDir != null && s3Dir != null) {
         Preconditions.checkArgument(msDir.getPath().equals(s3Dir.getPath()),
             String.format("The path from metadata store and s3 are different:" +


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[31/50] [abbrv] hadoop git commit: HDDS-170. Fix TestBlockDeletingService#testBlockDeletionTimeout. Contributed by Lokesh Jain.

Posted by gi...@apache.org.
HDDS-170. Fix TestBlockDeletingService#testBlockDeletionTimeout. Contributed by Lokesh Jain.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e305476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e305476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e305476

Branch: refs/heads/HADOOP-15461
Commit: 1e30547642c7c6c014745862dd06f90f091f90b6
Parents: e9ec3d7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:56:45 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:56:45 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/utils/BackgroundService.java  |  2 +-
 .../statemachine/background/BlockDeletingService.java    |  8 ++++----
 .../hadoop/ozone/container/ozoneimpl/OzoneContainer.java |  2 +-
 .../testutils/BlockDeletingServiceTestImpl.java          |  4 ++--
 .../ozone/container/common/TestBlockDeletingService.java | 11 +++++++----
 5 files changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
index 431da64..5718008 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
@@ -126,7 +126,7 @@ public abstract class BackgroundService {
         try {
           // Collect task results
           BackgroundTaskResult result = serviceTimeout > 0
-              ? taskResultFuture.get(serviceTimeout, TimeUnit.MILLISECONDS)
+              ? taskResultFuture.get(serviceTimeout, unit)
               : taskResultFuture.get();
           if (LOG.isDebugEnabled()) {
             LOG.debug("task execution result size {}", result.getSize());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
index 63f57b4..bff5913 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
@@ -83,10 +83,10 @@ public class BlockDeletingService extends BackgroundService{
   private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
 
   public BlockDeletingService(ContainerManager containerManager,
-      long serviceInterval, long serviceTimeout, Configuration conf) {
-    super("BlockDeletingService", serviceInterval,
-        TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE,
-        serviceTimeout);
+      long serviceInterval, long serviceTimeout, TimeUnit unit,
+      Configuration conf) {
+    super("BlockDeletingService", serviceInterval, unit,
+        BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
     this.containerManager = containerManager;
     this.conf = conf;
     this.blockLimitPerTask = conf.getInt(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 4156f5a..7931f6f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -122,7 +122,7 @@ public class OzoneContainer {
         OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
         OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
     this.blockDeletingService = new BlockDeletingService(manager,
-        svcInterval, serviceTimeout, ozoneConfig);
+        svcInterval, serviceTimeout, TimeUnit.MILLISECONDS, ozoneConfig);
 
     this.dispatcher = new Dispatcher(manager, this.ozoneConfig);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
index 86888aa..7c12945 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
@@ -44,8 +44,8 @@ public class BlockDeletingServiceTestImpl
 
   public BlockDeletingServiceTestImpl(ContainerManager containerManager,
       int serviceInterval, Configuration conf) {
-    super(containerManager, serviceInterval,
-        SERVICE_TIMEOUT_IN_MILLISECONDS, conf);
+    super(containerManager, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+        TimeUnit.MILLISECONDS, conf);
   }
 
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 0686e4e..8d01c80 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -280,10 +280,11 @@ public class TestBlockDeletingService {
     ContainerManager containerManager = createContainerManager(conf);
     createToDeleteBlocks(containerManager, conf, 1, 3, 1, chunksDir);
 
-    // set timeout value as 1ms to trigger timeout behavior
+    // set timeout value as 1ns to trigger timeout behavior
     long timeout  = 1;
-    BlockDeletingService svc =
-        new BlockDeletingService(containerManager, 1000, timeout, conf);
+    BlockDeletingService svc = new BlockDeletingService(containerManager,
+        TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
+        conf);
     svc.start();
 
     LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
@@ -303,7 +304,9 @@ public class TestBlockDeletingService {
     // test for normal case that doesn't have timeout limitation
     timeout  = 0;
     createToDeleteBlocks(containerManager, conf, 1, 3, 1, chunksDir);
-    svc =  new BlockDeletingService(containerManager, 1000, timeout, conf);
+    svc = new BlockDeletingService(containerManager,
+        TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
+        conf);
     svc.start();
 
     // get container meta data


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[37/50] [abbrv] hadoop git commit: HDDS-195. Create generic CommandWatcher utility. Contributed by Elek, Marton.

Posted by gi...@apache.org.
HDDS-195. Create generic CommandWatcher utility.
Contributed by Elek, Marton.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85627e2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85627e2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85627e2c

Branch: refs/heads/HADOOP-15461
Commit: 85627e2cba91a61d675d20cdb35e188c6c81e3f2
Parents: ddbff7c
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Jun 27 23:31:32 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 23:39:33 2018 -0700

----------------------------------------------------------------------
 .../hadoop/hdds/server/events/EventWatcher.java | 157 +++++++++++++
 .../server/events/IdentifiableEventPayload.java |  30 +++
 .../hdds/server/events/EventHandlerStub.java    |  38 ++++
 .../hdds/server/events/TestEventWatcher.java    | 220 +++++++++++++++++++
 4 files changed, 445 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
new file mode 100644
index 0000000..19fddde
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.ozone.lease.Lease;
+import org.apache.hadoop.ozone.lease.LeaseAlreadyExistException;
+import org.apache.hadoop.ozone.lease.LeaseExpiredException;
+import org.apache.hadoop.ozone.lease.LeaseManager;
+import org.apache.hadoop.ozone.lease.LeaseNotFoundException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Event watcher the (re)send a message after timeout.
+ * <p>
+ * Event watcher will send the tracked payload/event after a timeout period
+ * unless a confirmation from the original event (completion event) is arrived.
+ *
+ * @param <TIMEOUT_PAYLOAD>    The type of the events which are tracked.
+ * @param <COMPLETION_PAYLOAD> The type of event which could cancel the
+ *                             tracking.
+ */
+@SuppressWarnings("CheckStyle")
+public abstract class EventWatcher<TIMEOUT_PAYLOAD extends
+    IdentifiableEventPayload,
+    COMPLETION_PAYLOAD extends IdentifiableEventPayload> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(EventWatcher.class);
+
+  private final Event<TIMEOUT_PAYLOAD> startEvent;
+
+  private final Event<COMPLETION_PAYLOAD> completionEvent;
+
+  private final LeaseManager<UUID> leaseManager;
+
+  protected final Map<UUID, TIMEOUT_PAYLOAD> trackedEventsByUUID =
+      new ConcurrentHashMap<>();
+
+  protected final Set<TIMEOUT_PAYLOAD> trackedEvents = new HashSet<>();
+
+  public EventWatcher(Event<TIMEOUT_PAYLOAD> startEvent,
+      Event<COMPLETION_PAYLOAD> completionEvent,
+      LeaseManager<UUID> leaseManager) {
+    this.startEvent = startEvent;
+    this.completionEvent = completionEvent;
+    this.leaseManager = leaseManager;
+
+  }
+
+  public void start(EventQueue queue) {
+
+    queue.addHandler(startEvent, this::handleStartEvent);
+
+    queue.addHandler(completionEvent, (completionPayload, publisher) -> {
+      UUID uuid = completionPayload.getUUID();
+      try {
+        handleCompletion(uuid, publisher);
+      } catch (LeaseNotFoundException e) {
+        //It's already done. Too late, we already retried it.
+        //Not a real problem.
+        LOG.warn("Completion event without active lease. UUID={}", uuid);
+      }
+    });
+
+  }
+
+  private synchronized void handleStartEvent(TIMEOUT_PAYLOAD payload,
+      EventPublisher publisher) {
+    UUID identifier = payload.getUUID();
+    trackedEventsByUUID.put(identifier, payload);
+    trackedEvents.add(payload);
+    try {
+      Lease<UUID> lease = leaseManager.acquire(identifier);
+      try {
+        lease.registerCallBack(() -> {
+          handleTimeout(publisher, identifier);
+          return null;
+        });
+
+      } catch (LeaseExpiredException e) {
+        handleTimeout(publisher, identifier);
+      }
+    } catch (LeaseAlreadyExistException e) {
+      //No problem at all. But timer is not reset.
+    }
+  }
+
+  private synchronized void handleCompletion(UUID uuid,
+      EventPublisher publisher) throws LeaseNotFoundException {
+    leaseManager.release(uuid);
+    TIMEOUT_PAYLOAD payload = trackedEventsByUUID.remove(uuid);
+    trackedEvents.remove(payload);
+    onFinished(publisher, payload);
+  }
+
+  private synchronized void handleTimeout(EventPublisher publisher,
+      UUID identifier) {
+    TIMEOUT_PAYLOAD payload = trackedEventsByUUID.remove(identifier);
+    trackedEvents.remove(payload);
+    onTimeout(publisher, payload);
+  }
+
+
+  /**
+   * Check if a specific payload is in-progress.
+   */
+  public synchronized boolean contains(TIMEOUT_PAYLOAD payload) {
+    return trackedEvents.contains(payload);
+  }
+
+  public synchronized boolean remove(TIMEOUT_PAYLOAD payload) {
+    try {
+      leaseManager.release(payload.getUUID());
+    } catch (LeaseNotFoundException e) {
+      LOG.warn("Completion event without active lease. UUID={}",
+          payload.getUUID());
+    }
+    trackedEventsByUUID.remove(payload.getUUID());
+    return trackedEvents.remove(payload);
+
+  }
+
+  abstract void onTimeout(EventPublisher publisher, TIMEOUT_PAYLOAD payload);
+
+  abstract void onFinished(EventPublisher publisher, TIMEOUT_PAYLOAD payload);
+
+  public List<TIMEOUT_PAYLOAD> getTimeoutEvents(
+      Predicate<? super TIMEOUT_PAYLOAD> predicate) {
+    return trackedEventsByUUID.values().stream().filter(predicate)
+        .collect(Collectors.toList());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
new file mode 100644
index 0000000..e73e30f
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.UUID;
+
+/**
+ * Event with an additional unique identifier.
+ *
+ */
+public interface IdentifiableEventPayload {
+
+  UUID getUUID();
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
new file mode 100644
index 0000000..3f34a70
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Dummy class for testing to collect all the received events.
+ */
+public class EventHandlerStub<PAYLOAD> implements EventHandler<PAYLOAD> {
+
+  private List<PAYLOAD> receivedEvents = new ArrayList<>();
+
+  @Override
+  public void onMessage(PAYLOAD payload, EventPublisher publisher) {
+    receivedEvents.add(payload);
+  }
+
+  public List<PAYLOAD> getReceivedEvents() {
+    return receivedEvents;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
new file mode 100644
index 0000000..1731350
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+import org.apache.hadoop.ozone.lease.LeaseManager;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the basic functionality of event watcher.
+ */
+public class TestEventWatcher {
+
+  private static final TypedEvent<UnderreplicatedEvent> WATCH_UNDER_REPLICATED =
+      new TypedEvent<>(UnderreplicatedEvent.class);
+
+  private static final TypedEvent<UnderreplicatedEvent> UNDER_REPLICATED =
+      new TypedEvent<>(UnderreplicatedEvent.class);
+
+  private static final TypedEvent<ReplicationCompletedEvent>
+      REPLICATION_COMPLETED = new TypedEvent<>(ReplicationCompletedEvent.class);
+
+  LeaseManager<UUID> leaseManager;
+
+  @Before
+  public void startLeaseManager() {
+    leaseManager = new LeaseManager<>(2000l);
+    leaseManager.start();
+  }
+
+  @After
+  public void stopLeaseManager() {
+    leaseManager.shutdown();
+  }
+
+
+  @Test
+  public void testEventHandling() throws InterruptedException {
+
+    EventQueue queue = new EventQueue();
+
+    EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+        replicationWatcher = createEventWatcher();
+
+    EventHandlerStub<UnderreplicatedEvent> underReplicatedEvents =
+        new EventHandlerStub<>();
+
+    queue.addHandler(UNDER_REPLICATED, underReplicatedEvents);
+
+    replicationWatcher.start(queue);
+
+    UUID uuid1 = UUID.randomUUID();
+    UUID uuid2 = UUID.randomUUID();
+
+    queue.fireEvent(WATCH_UNDER_REPLICATED,
+        new UnderreplicatedEvent(uuid1, "C1"));
+
+    queue.fireEvent(WATCH_UNDER_REPLICATED,
+        new UnderreplicatedEvent(uuid2, "C2"));
+
+    Assert.assertEquals(0, underReplicatedEvents.getReceivedEvents().size());
+
+    Thread.sleep(1000);
+
+    queue.fireEvent(REPLICATION_COMPLETED,
+        new ReplicationCompletedEvent(uuid1, "C2", "D1"));
+
+    Assert.assertEquals(0, underReplicatedEvents.getReceivedEvents().size());
+
+    Thread.sleep(1500);
+
+    queue.processAll(1000L);
+
+    Assert.assertEquals(1, underReplicatedEvents.getReceivedEvents().size());
+    Assert.assertEquals(uuid2,
+        underReplicatedEvents.getReceivedEvents().get(0).UUID);
+
+  }
+
+  @Test
+  public void testInprogressFilter() throws InterruptedException {
+
+    EventQueue queue = new EventQueue();
+
+    EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+        replicationWatcher = createEventWatcher();
+
+    EventHandlerStub<UnderreplicatedEvent> underReplicatedEvents =
+        new EventHandlerStub<>();
+
+    queue.addHandler(UNDER_REPLICATED, underReplicatedEvents);
+
+    replicationWatcher.start(queue);
+
+    UnderreplicatedEvent event1 =
+        new UnderreplicatedEvent(UUID.randomUUID(), "C1");
+
+    queue.fireEvent(WATCH_UNDER_REPLICATED, event1);
+
+    queue.fireEvent(WATCH_UNDER_REPLICATED,
+        new UnderreplicatedEvent(UUID.randomUUID(), "C2"));
+
+    queue.fireEvent(WATCH_UNDER_REPLICATED,
+        new UnderreplicatedEvent(UUID.randomUUID(), "C1"));
+
+    queue.processAll(1000L);
+    Thread.sleep(1000L);
+    List<UnderreplicatedEvent> c1todo = replicationWatcher
+        .getTimeoutEvents(e -> e.containerId.equalsIgnoreCase("C1"));
+
+    Assert.assertEquals(2, c1todo.size());
+    Assert.assertTrue(replicationWatcher.contains(event1));
+    Thread.sleep(1500L);
+
+    c1todo = replicationWatcher
+        .getTimeoutEvents(e -> e.containerId.equalsIgnoreCase("C1"));
+    Assert.assertEquals(0, c1todo.size());
+    Assert.assertFalse(replicationWatcher.contains(event1));
+
+
+  }
+
+  private EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+  createEventWatcher() {
+    return new EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>(
+        WATCH_UNDER_REPLICATED, REPLICATION_COMPLETED, leaseManager) {
+
+      @Override
+      void onTimeout(EventPublisher publisher, UnderreplicatedEvent payload) {
+        publisher.fireEvent(UNDER_REPLICATED, payload);
+      }
+
+      @Override
+      void onFinished(EventPublisher publisher, UnderreplicatedEvent payload) {
+        //Good job. We did it.
+      }
+    };
+  }
+
+  private static class ReplicationCompletedEvent
+      implements IdentifiableEventPayload {
+
+    private final UUID UUID;
+
+    private final String containerId;
+
+    private final String datanodeId;
+
+    public ReplicationCompletedEvent(UUID UUID, String containerId,
+        String datanodeId) {
+      this.UUID = UUID;
+      this.containerId = containerId;
+      this.datanodeId = datanodeId;
+    }
+
+    public UUID getUUID() {
+      return UUID;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      ReplicationCompletedEvent that = (ReplicationCompletedEvent) o;
+      return Objects.equals(containerId, that.containerId) && Objects
+          .equals(datanodeId, that.datanodeId);
+    }
+
+    @Override
+    public int hashCode() {
+
+      return Objects.hash(containerId, datanodeId);
+    }
+  }
+
+  private static class UnderreplicatedEvent
+
+      implements IdentifiableEventPayload {
+
+    private final UUID UUID;
+
+    private final String containerId;
+
+    public UnderreplicatedEvent(UUID UUID, String containerId) {
+      this.containerId = containerId;
+      this.UUID = UUID;
+    }
+
+    public UUID getUUID() {
+      return UUID;
+    }
+  }
+
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[38/50] [abbrv] hadoop git commit: HDFS-13705:The native ISA-L library loading failure should be made warning rather than an error message. Contributed by Shashikant Banerjee.

Posted by gi...@apache.org.
HDFS-13705:The native ISA-L library loading failure should be made warning rather than an error message. Contributed by Shashikant Banerjee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3fa83a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3fa83a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3fa83a4

Branch: refs/heads/HADOOP-15461
Commit: d3fa83a44b01c85f39bfb4deaf2972912ac61ca3
Parents: 85627e2
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Jun 28 09:21:56 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Jun 28 09:21:56 2018 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fa83a4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
index 3d6867a..ec317ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
@@ -46,7 +46,7 @@ public final class ErasureCodeNative {
         loadLibrary();
       } catch (Throwable t) {
         problem = "Loading ISA-L failed: " + t.getMessage();
-        LOG.error("Loading ISA-L failed", t);
+        LOG.warn(problem);
       }
       LOADING_FAILURE_REASON = problem;
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[34/50] [abbrv] hadoop git commit: HADOOP-15495. Upgrade commons-lang version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by Takanobu Asanuma.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 0ab8696..116827d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -60,7 +60,7 @@ import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 49981ed..f0ffb44 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -23,7 +23,7 @@ import com.google.common.base.Preconditions;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index ac10e08..527697f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -39,7 +39,7 @@ import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileStatus;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index bfac975..9e2f34d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.s3native;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -31,7 +31,7 @@ import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.util.Objects;
 
-import static org.apache.commons.lang.StringUtils.equalsIgnoreCase;
+import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase;
 
 /**
  * Class to aid logging in to S3 endpoints.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index f1799ac..aa6b5d8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -22,8 +22,8 @@ import com.amazonaws.ClientConfiguration;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.S3ClientOptions;
 
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.reflect.FieldUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
index 50c9fb5..a8a78f6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
index 4c953bd..c170830 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 
 /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index d259bf1..869997b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.s3a;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
index b8610d6..90e8894 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
@@ -28,7 +28,7 @@ import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 3e149a6..79e8a69 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -42,7 +42,7 @@ import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 12c2e3f..a68e6ac 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -25,7 +25,7 @@ import java.net.URISyntaxException;
 
 import com.microsoft.azure.datalake.store.oauth2.DeviceCodeTokenProvider;
 import com.microsoft.azure.datalake.store.oauth2.MsiTokenProvider;
-import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
 import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 9396a51..197ab22 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -41,7 +41,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index 9a85308..5f051ef 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -44,7 +44,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.commons.codec.binary.Base64;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
index aa7ac2e..850e552 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.azure;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index dfc881a..5202762 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -48,7 +48,7 @@ import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
index 68ddcdf..6e98755 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
@@ -36,7 +36,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.fs.Syncable;
 import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
index ea77510..76ced3b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.azure.security.Constants;
 import org.apache.hadoop.io.retry.RetryPolicy;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
index a0204be..f4ec172 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.azure;
 
-import org.apache.commons.lang.Validate;
+import org.apache.commons.lang3.Validate;
 import org.apache.hadoop.fs.azure.security.Constants;
 import org.apache.hadoop.fs.azure.security.SpnegoToken;
 import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
index 0aa9393..f54a2e1 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
@@ -29,7 +29,7 @@ import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.util.Arrays;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
index 29611bf..4e88b45 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
@@ -22,7 +22,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.util.Arrays;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index d5f6437..1739cff 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -36,7 +36,7 @@ import java.util.TimeZone;
 import java.util.List;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.net.URLCodec;
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.http.client.utils.URIBuilder;
 
 import com.microsoft.azure.storage.AccessCondition;
@@ -339,7 +339,7 @@ public class MockStorageInterface extends StorageInterface {
 
     @Override
     public StorageUri getStorageUri() {
-      throw new NotImplementedException();
+      throw new NotImplementedException("Code is not implemented");
     }
   }
 
@@ -590,20 +590,20 @@ public class MockStorageInterface extends StorageInterface {
     @Override
     public void create(long length, BlobRequestOptions options,
         OperationContext opContext) throws StorageException {
-      throw new NotImplementedException();
+      throw new NotImplementedException("Code is not implemented");
     }
 
     @Override
     public void uploadPages(InputStream sourceStream, long offset, long length,
         BlobRequestOptions options, OperationContext opContext)
         throws StorageException, IOException {
-      throw new NotImplementedException();
+      throw new NotImplementedException("Code is not implemented");
     }
 
     @Override
     public ArrayList<PageRange> downloadPageRanges(BlobRequestOptions options,
         OperationContext opContext) throws StorageException {
-      throw new NotImplementedException();
+      throw new NotImplementedException("Code is not implemented");
     }
 
     @Override
@@ -622,7 +622,7 @@ public class MockStorageInterface extends StorageInterface {
 
     @Override
     public StorageUri getStorageUri() {
-        throw new NotImplementedException();
+        throw new NotImplementedException("Code is not implemented");
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
index 820ce4f..b8cf5ba 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.azure;
 
 import com.microsoft.azure.storage.blob.BlockEntry;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
index 2fbbcd1..8d2a104 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
@@ -28,7 +28,7 @@ import org.junit.internal.AssumptionViolatedException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index ea99016..9db0eb5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.tools;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 668b594..e49feb5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -28,7 +28,7 @@ import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index faa4aa2..c486bdb 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -22,7 +22,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.EnumSet;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
index 9568171..cf6da25 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.mapred.gridmix;
 
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
index 8f9d434..973838a 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.mapred.gridmix;
 
 import java.io.IOException;
 
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -76,7 +76,7 @@ class ExecutionSummarizer implements StatListener<JobStats> {
     startTime = System.currentTimeMillis();
     // flatten the args string and store it
     commandLineArgs = 
-      org.apache.commons.lang.StringUtils.join(args, ' '); 
+      org.apache.commons.lang3.StringUtils.join(args, ' ');
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
index 4271742..73662bf 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.mapred.gridmix;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
index 877d434..494b9a1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
@@ -21,7 +21,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
index afafd55..2accbac 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
@@ -30,7 +30,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.lang.CharSet;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.resourceestimator.common.api.RecurrenceId;
 import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
index 5856626..e6d09dc 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.tools.rumen.anonymization;
 
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * Utility class to handle commonly performed tasks in a 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
index 1c92caf..02fd48a 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
@@ -25,7 +25,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapreduce.MRJobConfig;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
index bd50ae0..e1f6da5 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
@@ -25,7 +25,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[30/50] [abbrv] hadoop git commit: HDDS-186. Create under replicated queue. Contributed by Ajay Kumar.

Posted by gi...@apache.org.
HDDS-186. Create under replicated queue. Contributed by Ajay Kumar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9ec3d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9ec3d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9ec3d78

Branch: refs/heads/HADOOP-15461
Commit: e9ec3d78f520a8543dc77d763d4b358aa608bae8
Parents: 56a4cdb
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:35:30 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:35:30 2018 -0700

----------------------------------------------------------------------
 .../container/replication/ReplicationQueue.java |  76 +++++++++++
 .../replication/ReplicationRequest.java         | 106 +++++++++++++++
 .../container/replication/package-info.java     |  23 ++++
 .../replication/TestReplicationQueue.java       | 134 +++++++++++++++++++
 .../container/replication/package-info.java     |  23 ++++
 5 files changed, 362 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
new file mode 100644
index 0000000..e0a2351
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+
+/**
+ * Priority queue to handle under-replicated and over replicated containers
+ * in ozone. ReplicationManager will consume these messages and decide
+ * accordingly.
+ */
+public class ReplicationQueue {
+
+  private final Queue<ReplicationRequest> queue;
+
+  ReplicationQueue() {
+    queue = new PriorityQueue<>();
+  }
+
+  public synchronized boolean add(ReplicationRequest repObj) {
+    if (this.queue.contains(repObj)) {
+      // Remove the earlier message and insert this one
+      this.queue.remove(repObj);
+    }
+    return this.queue.add(repObj);
+  }
+
+  public synchronized boolean remove(ReplicationRequest repObj) {
+    return queue.remove(repObj);
+  }
+
+  /**
+   * Retrieves, but does not remove, the head of this queue,
+   * or returns {@code null} if this queue is empty.
+   *
+   * @return the head of this queue, or {@code null} if this queue is empty
+   */
+  public synchronized ReplicationRequest peek() {
+    return queue.peek();
+  }
+
+  /**
+   * Retrieves and removes the head of this queue,
+   * or returns {@code null} if this queue is empty.
+   *
+   * @return the head of this queue, or {@code null} if this queue is empty
+   */
+  public synchronized ReplicationRequest poll() {
+    return queue.poll();
+  }
+
+  public synchronized boolean removeAll(List<ReplicationRequest> repObjs) {
+    return queue.removeAll(repObjs);
+  }
+
+  public int size() {
+    return queue.size();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
new file mode 100644
index 0000000..a6ccce1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * Wrapper class for hdds replication queue. Implements its natural
+ * ordering for priority queue.
+ */
+public class ReplicationRequest implements Comparable<ReplicationRequest>,
+    Serializable {
+  private final long containerId;
+  private final short replicationCount;
+  private final short expecReplicationCount;
+  private final long timestamp;
+
+  public ReplicationRequest(long containerId, short replicationCount,
+      long timestamp, short expecReplicationCount) {
+    this.containerId = containerId;
+    this.replicationCount = replicationCount;
+    this.timestamp = timestamp;
+    this.expecReplicationCount = expecReplicationCount;
+  }
+
+  /**
+   * Compares this object with the specified object for order.  Returns a
+   * negative integer, zero, or a positive integer as this object is less
+   * than, equal to, or greater than the specified object.
+   * @param o the object to be compared.
+   * @return a negative integer, zero, or a positive integer as this object
+   * is less than, equal to, or greater than the specified object.
+   * @throws NullPointerException if the specified object is null
+   * @throws ClassCastException   if the specified object's type prevents it
+   *                              from being compared to this object.
+   */
+  @Override
+  public int compareTo(ReplicationRequest o) {
+    if (o == null) {
+      return 1;
+    }
+    if (this == o) {
+      return 0;
+    }
+    int retVal = Integer
+        .compare(getReplicationCount() - getExpecReplicationCount(),
+            o.getReplicationCount() - o.getExpecReplicationCount());
+    if (retVal != 0) {
+      return retVal;
+    }
+    return Long.compare(getTimestamp(), o.getTimestamp());
+  }
+
+  @Override
+  public int hashCode() {
+    return new HashCodeBuilder(91, 1011)
+        .append(getContainerId())
+        .toHashCode();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ReplicationRequest that = (ReplicationRequest) o;
+    return new EqualsBuilder().append(getContainerId(), that.getContainerId())
+        .isEquals();
+  }
+
+  public long getContainerId() {
+    return containerId;
+  }
+
+  public short getReplicationCount() {
+    return replicationCount;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public short getExpecReplicationCount() {
+    return expecReplicationCount;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..7f335e3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Ozone Container replicaton related classes.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
new file mode 100644
index 0000000..6d74c68
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.Random;
+import java.util.UUID;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for ReplicationQueue.
+ */
+public class TestReplicationQueue {
+
+  private ReplicationQueue replicationQueue;
+  private Random random;
+
+  @Before
+  public void setUp() {
+    replicationQueue = new ReplicationQueue();
+    random = new Random();
+  }
+
+  @Test
+  public void testDuplicateAddOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationRequest obj1, obj2, obj3;
+    long time = Time.monotonicNow();
+    obj1 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
+    obj2 = new ReplicationRequest(contId, (short) 2, time + 1, (short) 3);
+    obj3 = new ReplicationRequest(contId, (short) 1, time+2, (short) 3);
+
+    replicationQueue.add(obj1);
+    replicationQueue.add(obj2);
+    replicationQueue.add(obj3);
+    Assert.assertEquals("Should add only 1 msg as second one is duplicate",
+        1, replicationQueue.size());
+    ReplicationRequest temp = replicationQueue.poll();
+    Assert.assertEquals(temp, obj3);
+  }
+
+  @Test
+  public void testPollOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationRequest msg1, msg2, msg3, msg4, msg5;
+    msg1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
+        (short) 3);
+    long time = Time.monotonicNow();
+    msg2 = new ReplicationRequest(contId + 1, (short) 4, time, (short) 3);
+    msg3 = new ReplicationRequest(contId + 2, (short) 0, time, (short) 3);
+    msg4 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
+    // Replication message for same container but different nodeId
+    msg5 = new ReplicationRequest(contId + 1, (short) 2, time, (short) 3);
+
+    replicationQueue.add(msg1);
+    replicationQueue.add(msg2);
+    replicationQueue.add(msg3);
+    replicationQueue.add(msg4);
+    replicationQueue.add(msg5);
+    Assert.assertEquals("Should have 3 objects",
+        3, replicationQueue.size());
+
+    // Since Priority queue orders messages according to replication count,
+    // message with lowest replication should be first
+    ReplicationRequest temp;
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 2 objects",
+        2, replicationQueue.size());
+    Assert.assertEquals(temp, msg3);
+
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 1 objects",
+        1, replicationQueue.size());
+    Assert.assertEquals(temp, msg5);
+
+    // Message 2 should be ordered before message 5 as both have same replication
+    // number but message 2 has earlier timestamp.
+    temp = replicationQueue.poll();
+    Assert.assertEquals("Should have 0 objects",
+        replicationQueue.size(), 0);
+    Assert.assertEquals(temp, msg4);
+  }
+
+  @Test
+  public void testRemoveOp() {
+    long contId = random.nextLong();
+    String nodeId = UUID.randomUUID().toString();
+    ReplicationRequest obj1, obj2, obj3;
+    obj1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
+        (short) 3);
+    obj2 = new ReplicationRequest(contId + 1, (short) 2, Time.monotonicNow(),
+        (short) 3);
+    obj3 = new ReplicationRequest(contId + 2, (short) 3, Time.monotonicNow(),
+        (short) 3);
+
+    replicationQueue.add(obj1);
+    replicationQueue.add(obj2);
+    replicationQueue.add(obj3);
+    Assert.assertEquals("Should have 3 objects",
+        3, replicationQueue.size());
+
+    replicationQueue.remove(obj3);
+    Assert.assertEquals("Should have 2 objects",
+        2, replicationQueue.size());
+
+    replicationQueue.remove(obj2);
+    Assert.assertEquals("Should have 1 objects",
+        1, replicationQueue.size());
+
+    replicationQueue.remove(obj1);
+    Assert.assertEquals("Should have 0 objects",
+        0, replicationQueue.size());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5b1fd0f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * SCM Testing and Mocking Utils.
+ */
+package org.apache.hadoop.ozone.container.replication;
+// Test classes for Replication functionality.
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[25/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index 2bd43fb..edc0d7b 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.scm.cli;
 
 import com.google.common.base.Preconditions;
 import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
 import org.apache.commons.cli.BasicParser;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
@@ -60,13 +59,11 @@ import java.sql.Statement;
 import java.util.HashSet;
 import java.util.Set;
 
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
 
 /**
@@ -111,15 +108,6 @@ public class SQLCLI  extends Configured implements Tool {
   private static final String INSERT_CONTAINER_MEMBERS =
       "INSERT INTO containerMembers (containerName, datanodeUUID) " +
           "VALUES (\"%s\", \"%s\")";
-  // for nodepool.db
-  private static final String CREATE_NODE_POOL =
-      "CREATE TABLE nodePool (" +
-          "datanodeUUID TEXT NOT NULL," +
-          "poolName TEXT NOT NULL," +
-          "PRIMARY KEY(datanodeUUID, poolName))";
-  private static final String INSERT_NODE_POOL =
-      "INSERT INTO nodePool (datanodeUUID, poolName) " +
-          "VALUES (\"%s\", \"%s\")";
   // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
   // for openContainer.db
   private static final String CREATE_OPEN_CONTAINER =
@@ -285,9 +273,6 @@ public class SQLCLI  extends Configured implements Tool {
     if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
       LOG.info("Converting container DB");
       convertContainerDB(dbPath, outPath);
-    }  else if (dbName.toString().equals(NODEPOOL_DB)) {
-      LOG.info("Converting node pool DB");
-      convertNodePoolDB(dbPath, outPath);
     } else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
       LOG.info("Converting open container DB");
       convertOpenContainerDB(dbPath, outPath);
@@ -543,66 +528,7 @@ public class SQLCLI  extends Configured implements Tool {
     }
     LOG.info("Insertion completed.");
   }
-  /**
-   * Converts nodePool.db to sqlite. The schema of sql db:
-   * two tables, nodePool and datanodeInfo (the same datanode Info as for
-   * container.db).
-   *
-   * nodePool
-   * ---------------------------------------------------------
-   * datanodeUUID* | poolName*
-   * ---------------------------------------------------------
-   *
-   * datanodeInfo:
-   * ---------------------------------------------------------
-   * hostname | datanodeUUid* | xferPort | ipcPort
-   * ---------------------------------------------------------
-   *
-   * --------------------------------
-   * |containerPort
-   * --------------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
-    LOG.info("Create table for sql node pool db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_NODE_POOL);
-      executeSQL(conn, CREATE_DATANODE_INFO);
 
-      dbStore.iterate(null, (key, value) -> {
-        DatanodeDetails nodeId = DatanodeDetails
-            .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
-                .PARSER.parseFrom(key));
-        String blockPool = DFSUtil.bytes2String(value);
-        try {
-          insertNodePoolDB(conn, blockPool, nodeId);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  private void insertNodePoolDB(Connection conn, String blockPool,
-      DatanodeDetails datanodeDetails) throws SQLException {
-    String insertNodePool = String.format(INSERT_NODE_POOL,
-        datanodeDetails.getUuidString(), blockPool);
-    executeSQL(conn, insertNodePool);
-
-    String insertDatanodeDetails = String
-        .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
-            datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
-            datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
-                .getValue());
-    executeSQL(conn, insertDatanodeDetails);
-  }
 
   /**
    * Convert openContainer.db to sqlite db file. This is rather simple db,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[12/50] [abbrv] hadoop git commit: HADOOP-15458. TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed by Xiao Liang.

Posted by gi...@apache.org.
HADOOP-15458. TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed by Xiao Liang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abc3e4ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abc3e4ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abc3e4ba

Branch: refs/heads/HADOOP-15461
Commit: abc3e4bad905efde5a4881e8a072c68f6e910ade
Parents: 1ba4e62
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jun 25 09:50:27 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Jun 25 09:50:27 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/TestLocalFileSystem.java   | 23 ++++++++++----------
 1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/abc3e4ba/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
index 0e337b4..d5622af 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
@@ -689,17 +689,18 @@ public class TestLocalFileSystem {
     // and permission
     FSDataOutputStreamBuilder builder =
         fileSys.createFile(path);
-    builder.build();
-    Assert.assertEquals("Should be default block size",
-        builder.getBlockSize(), fileSys.getDefaultBlockSize());
-    Assert.assertEquals("Should be default replication factor",
-        builder.getReplication(), fileSys.getDefaultReplication());
-    Assert.assertEquals("Should be default buffer size",
-        builder.getBufferSize(),
-        fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
-            IO_FILE_BUFFER_SIZE_DEFAULT));
-    Assert.assertEquals("Should be default permission",
-        builder.getPermission(), FsPermission.getFileDefault());
+    try (FSDataOutputStream stream = builder.build()) {
+      Assert.assertEquals("Should be default block size",
+          builder.getBlockSize(), fileSys.getDefaultBlockSize());
+      Assert.assertEquals("Should be default replication factor",
+          builder.getReplication(), fileSys.getDefaultReplication());
+      Assert.assertEquals("Should be default buffer size",
+          builder.getBufferSize(),
+          fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+              IO_FILE_BUFFER_SIZE_DEFAULT));
+      Assert.assertEquals("Should be default permission",
+          builder.getPermission(), FsPermission.getFileDefault());
+    }
 
     // Test set 0 to replication, block size and buffer size
     builder = fileSys.createFile(path);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[06/50] [abbrv] hadoop git commit: HDDS-184. Upgrade common-langs version to 3.7 in hadoop-tools/hadoop-ozone. Contributed by Takanobu Asanuma.

Posted by gi...@apache.org.
HDDS-184. Upgrade common-langs version to 3.7 in hadoop-tools/hadoop-ozone.
Contributed by Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca14fec0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca14fec0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca14fec0

Branch: refs/heads/HADOOP-15461
Commit: ca14fec02fb14e1b708f266bc715e84ae9784d6f
Parents: 8a32bc3
Author: Anu Engineer <ae...@apache.org>
Authored: Sat Jun 23 15:49:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sat Jun 23 15:49:44 2018 -0700

----------------------------------------------------------------------
 .../ozone/genconf/TestGenerateOzoneRequiredConfigurations.java     | 2 +-
 .../src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java  | 2 +-
 .../java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java    | 2 +-
 .../java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java   | 2 +-
 .../java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
index cfd1159..c2f5eb7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.ozone.genconf;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.hamcrest.CoreMatchers;
 import org.junit.AfterClass;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 0ff1d50..6906a9d 100644
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index b82c4a1..ad21f28 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.ozone;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FSDataOutputStream;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index 5a7cb4f..a225702 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -31,7 +31,7 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.After;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index 176b614..8417e46 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.ozone.contract;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.fs.FileSystem;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[17/50] [abbrv] hadoop git commit: YARN-8438. TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.

Posted by gi...@apache.org.
YARN-8438. TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35ec9401
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35ec9401
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35ec9401

Branch: refs/heads/HADOOP-15461
Commit: 35ec9401e829bfa10994790659a26b0babacae35
Parents: 7a3c6e9
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Mon Jun 25 09:23:11 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Mon Jun 25 15:47:54 2018 -0700

----------------------------------------------------------------------
 .../containermanager/container/TestContainer.java            | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/35ec9401/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 1a263ee..edf26d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -116,7 +116,7 @@ public class TestContainer {
   final NodeManagerMetrics metrics = NodeManagerMetrics.create();
   final Configuration conf = new YarnConfiguration();
   final String FAKE_LOCALIZATION_ERROR = "Fake localization error";
-  
+
   /**
    * Verify correct container request events sent to localizer.
    */
@@ -591,9 +591,8 @@ public class TestContainer {
       Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
           containerMetrics.exitCode.value());
       Assert.assertTrue(containerMetrics.startTime.value() > 0);
-      Assert.assertTrue(
-          containerMetrics.finishTime.value() > containerMetrics.startTime
-              .value());
+      Assert.assertTrue(containerMetrics.finishTime.value() >=
+              containerMetrics.startTime.value());
       Assert.assertEquals(ContainerEventType.KILL_CONTAINER,
           wc.initStateToEvent.get(ContainerState.NEW));
       Assert.assertEquals(ContainerState.DONE,
@@ -1612,4 +1611,5 @@ public class TestContainer {
       return ((ContainerImpl)c).getRetryPolicy();
     }
   }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[24/50] [abbrv] hadoop git commit: YARN-8401. [UI2] new ui is not accessible with out internet connection. Contributed by Bibin A Chundatt.

Posted by gi...@apache.org.
YARN-8401. [UI2] new ui is not accessible with out internet connection. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbaff369
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbaff369
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbaff369

Branch: refs/heads/HADOOP-15461
Commit: fbaff369e9b9022723a7b2c6f25e71122a8f8a15
Parents: bedc4fe
Author: Sunil G <su...@apache.org>
Authored: Wed Jun 27 10:35:15 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Wed Jun 27 10:35:15 2018 -0700

----------------------------------------------------------------------
 .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbaff369/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
index ddb8532..ac74d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
@@ -16,10 +16,6 @@
 * limitations under the License.
 -->
 
-<!DOCTYPE web-app PUBLIC
- "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
- "http://java.sun.com/dtd/web-app_2_3.dtd" >
-
-<web-app>
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
   <display-name>YARN UI</display-name>
 </web-app>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[42/50] [abbrv] hadoop git commit: HDDS-178: DN should update transactionId on block delete. Contributed by Lokesh Jain.

Posted by gi...@apache.org.
HDDS-178: DN should update transactionId on block delete. Contributed by Lokesh Jain.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4d7227a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4d7227a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4d7227a

Branch: refs/heads/HADOOP-15461
Commit: e4d7227aad586f055b47bdc90c65361f9fb23146
Parents: 852ca6b
Author: Nanda kumar <na...@apache.org>
Authored: Fri Jun 29 04:37:16 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Jun 29 04:37:16 2018 +0530

----------------------------------------------------------------------
 .../common/impl/ContainerManagerImpl.java       |   7 +-
 .../common/interfaces/ContainerManager.java     |   2 +
 .../DeleteBlocksCommandHandler.java             |  29 ++-
 .../commandhandler/TestBlockDeletion.java       | 211 +++++++++++++++++++
 4 files changed, 243 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 02572a8..e81f1c6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -1111,5 +1111,10 @@ public class ContainerManagerImpl implements ContainerManager {
     return cData.getKeyCount();
   }
 
-
+  @Override
+  public void updateDeleteTransactionId(long containerId,
+      long deleteTransactionId) {
+    containerMap.get(containerId)
+        .updateDeleteTransactionId(deleteTransactionId);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
index 49b68dc..cf68b08 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
@@ -264,4 +264,6 @@ public interface ContainerManager extends RwLock {
    */
   long getNumKeys(long containerId);
 
+  void updateDeleteTransactionId(long containerId, long deleteTransactionId);
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index f954d98..d215da9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -158,6 +158,13 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
           containerInfo.getDBPath());
     }
 
+    if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) {
+      LOG.debug(String.format("Ignoring delete blocks for containerId: %d."
+              + " Outdated delete transactionId %d < %d", containerId,
+          delTX.getTxID(), containerInfo.getDeleteTransactionId()));
+      return;
+    }
+
     int newDeletionBlocks = 0;
     MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
     for (Long blk : delTX.getLocalIDList()) {
@@ -165,10 +172,20 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
       byte[] blkBytes = Longs.toByteArray(blk);
       byte[] blkInfo = containerDB.get(blkBytes);
       if (blkInfo != null) {
+        byte[] deletingKeyBytes =
+            DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk);
+        byte[] deletedKeyBytes =
+            DFSUtil.string2Bytes(OzoneConsts.DELETED_KEY_PREFIX + blk);
+        if (containerDB.get(deletingKeyBytes) != null
+            || containerDB.get(deletedKeyBytes) != null) {
+          LOG.debug(String.format(
+              "Ignoring delete for block %d in container %d."
+                  + " Entry already added.", blk, containerId));
+          continue;
+        }
         // Found the block in container db,
         // use an atomic update to change its state to deleting.
-        batch.put(DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk),
-            blkInfo);
+        batch.put(deletingKeyBytes, blkInfo);
         batch.delete(blkBytes);
         try {
           containerDB.writeBatch(batch);
@@ -186,11 +203,13 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
         LOG.debug("Block {} not found or already under deletion in"
                 + " container {}, skip deleting it.", blk, containerId);
       }
-      containerDB.put(DFSUtil.string2Bytes(
-          OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
-          Longs.toByteArray(delTX.getTxID()));
     }
 
+    containerDB.put(DFSUtil.string2Bytes(
+        OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
+        Longs.toByteArray(delTX.getTxID()));
+    containerManager
+        .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID());
     // update pending deletion blocks count in in-memory container status
     containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
new file mode 100644
index 0000000..43e3f50
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
+import org.apache.hadoop.ozone.ksm.KeySpaceManager;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.utils.MetadataStore;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+
+public class TestBlockDeletion {
+  private static OzoneConfiguration conf = null;
+  private static ObjectStore store;
+  private static ContainerManagerImpl dnContainerManager = null;
+  private static StorageContainerManager scm = null;
+  private static KeySpaceManager ksm = null;
+  private static Set<Long> containerIdsWithDeletedBlocks;
+
+  @BeforeClass
+  public static void init() throws Exception {
+    conf = new OzoneConfiguration();
+
+    String path =
+        GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName());
+    File baseDir = new File(path);
+    baseDir.mkdirs();
+
+    path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
+        OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
+
+    conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
+    conf.setQuietMode(false);
+    conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
+        TimeUnit.MILLISECONDS);
+
+    MiniOzoneCluster cluster =
+        MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+    cluster.waitForClusterToBeReady();
+    store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
+    dnContainerManager =
+        (ContainerManagerImpl) cluster.getHddsDatanodes().get(0)
+            .getDatanodeStateMachine().getContainer().getContainerManager();
+    ksm = cluster.getKeySpaceManager();
+    scm = cluster.getStorageContainerManager();
+    containerIdsWithDeletedBlocks = new HashSet<>();
+  }
+
+  @Test(timeout = 60000)
+  public void testBlockDeletion()
+      throws IOException, InterruptedException {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+
+    String value = RandomStringUtils.random(1000000);
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+
+    String keyName = UUID.randomUUID().toString();
+
+    OzoneOutputStream out = bucket.createKey(keyName, value.getBytes().length,
+        ReplicationType.STAND_ALONE, ReplicationFactor.ONE);
+    out.write(value.getBytes());
+    out.close();
+
+    KsmKeyArgs keyArgs = new KsmKeyArgs.Builder().setVolumeName(volumeName)
+        .setBucketName(bucketName).setKeyName(keyName).setDataSize(0)
+        .setType(HddsProtos.ReplicationType.STAND_ALONE)
+        .setFactor(HddsProtos.ReplicationFactor.ONE).build();
+    List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroupList =
+        ksm.lookupKey(keyArgs).getKeyLocationVersions();
+
+    // verify key blocks were created in DN.
+    Assert.assertTrue(verifyBlocksCreated(ksmKeyLocationInfoGroupList));
+    // No containers with deleted blocks
+    Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty());
+    // Delete transactionIds for the containers should be 0
+    matchContainerTransactionIds();
+    ksm.deleteKey(keyArgs);
+    Thread.sleep(5000);
+    // The blocks should be deleted in the DN.
+    Assert.assertTrue(verifyBlocksDeleted(ksmKeyLocationInfoGroupList));
+
+    // Few containers with deleted blocks
+    Assert.assertTrue(!containerIdsWithDeletedBlocks.isEmpty());
+    // Containers in the DN and SCM should have same delete transactionIds
+    matchContainerTransactionIds();
+  }
+
+  private void matchContainerTransactionIds() throws IOException {
+    List<ContainerData> containerDataList = new ArrayList<>();
+    dnContainerManager.listContainer(0, 10000, containerDataList);
+    for (ContainerData containerData : containerDataList) {
+      long containerId = containerData.getContainerID();
+      if (containerIdsWithDeletedBlocks.contains(containerId)) {
+        Assert.assertTrue(
+            scm.getContainerInfo(containerId).getDeleteTransactionId() > 0);
+      } else {
+        Assert.assertEquals(
+            scm.getContainerInfo(containerId).getDeleteTransactionId(), 0);
+      }
+      Assert.assertEquals(dnContainerManager.readContainer(containerId)
+              .getDeleteTransactionId(),
+          scm.getContainerInfo(containerId).getDeleteTransactionId());
+    }
+  }
+
+  private boolean verifyBlocksCreated(
+      List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+      throws IOException {
+    return performOperationOnKeyContainers((blockID) -> {
+      try {
+        MetadataStore db = KeyUtils.getDB(
+            dnContainerManager.getContainerMap().get(blockID.getContainerID()),
+            conf);
+        Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID())));
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+    }, ksmKeyLocationInfoGroups);
+  }
+
+  private boolean verifyBlocksDeleted(
+      List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+      throws IOException {
+    return performOperationOnKeyContainers((blockID) -> {
+      try {
+        MetadataStore db = KeyUtils.getDB(
+            dnContainerManager.getContainerMap().get(blockID.getContainerID()),
+            conf);
+        Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID())));
+        Assert.assertNull(db.get(DFSUtil.string2Bytes(
+            OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID())));
+        Assert.assertNotNull(DFSUtil.string2Bytes(
+            OzoneConsts.DELETED_KEY_PREFIX + blockID.getLocalID()));
+        containerIdsWithDeletedBlocks.add(blockID.getContainerID());
+      } catch (IOException e) {
+        e.printStackTrace();
+      }
+    }, ksmKeyLocationInfoGroups);
+  }
+
+  private boolean performOperationOnKeyContainers(Consumer<BlockID> consumer,
+      List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+      throws IOException {
+
+    try {
+      for (KsmKeyLocationInfoGroup ksmKeyLocationInfoGroup : ksmKeyLocationInfoGroups) {
+        List<KsmKeyLocationInfo> ksmKeyLocationInfos =
+            ksmKeyLocationInfoGroup.getLocationList();
+        for (KsmKeyLocationInfo ksmKeyLocationInfo : ksmKeyLocationInfos) {
+          BlockID blockID = ksmKeyLocationInfo.getBlockID();
+          consumer.accept(blockID);
+        }
+      }
+    } catch (Error e) {
+      e.printStackTrace();
+      return false;
+    }
+    return true;
+  }
+}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[46/50] [abbrv] hadoop git commit: YARN-8451. Multiple NM heartbeat thread created when a slow NM resync with RM. Contributed by Botong Huang

Posted by gi...@apache.org.
YARN-8451. Multiple NM heartbeat thread created when a slow NM resync with RM. Contributed by Botong Huang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10047014
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10047014
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10047014

Branch: refs/heads/HADOOP-15461
Commit: 100470140d86eede0fa240a9aa93226f274ee4f5
Parents: a820738
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jun 29 13:06:28 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jun 29 13:06:28 2018 -0500

----------------------------------------------------------------------
 .../yarn/server/nodemanager/NodeManager.java    | 66 +++++++++++++-------
 .../nodemanager/TestNodeManagerResync.java      | 56 +++++++++++++++++
 2 files changed, 98 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/10047014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 2748a8f..c8234bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -128,6 +128,7 @@ public class NodeManager extends CompositeService
   // the NM collector service is set only if the timeline service v.2 is enabled
   private NMCollectorService nmCollectorService;
   private NodeStatusUpdater nodeStatusUpdater;
+  private AtomicBoolean resyncingWithRM = new AtomicBoolean(false);
   private NodeResourceMonitor nodeResourceMonitor;
   private static CompositeServiceShutdownHook nodeManagerShutdownHook;
   private NMStateStoreService nmStore = null;
@@ -393,7 +394,7 @@ public class NodeManager extends CompositeService
     addService(del);
 
     // NodeManager level dispatcher
-    this.dispatcher = new AsyncDispatcher("NM Event dispatcher");
+    this.dispatcher = createNMDispatcher();
 
     nodeHealthChecker =
         new NodeHealthCheckerService(
@@ -517,31 +518,41 @@ public class NodeManager extends CompositeService
   }
 
   protected void resyncWithRM() {
-    //we do not want to block dispatcher thread here
-    new Thread() {
-      @Override
-      public void run() {
-        try {
-          if (!rmWorkPreservingRestartEnabled) {
-            LOG.info("Cleaning up running containers on resync");
-            containerManager.cleanupContainersOnNMResync();
-            // Clear all known collectors for resync.
-            if (context.getKnownCollectors() != null) {
-              context.getKnownCollectors().clear();
+    // Create a thread for resync because we do not want to block dispatcher
+    // thread here. Also use locking to make sure only one thread is running at
+    // a time.
+    if (this.resyncingWithRM.getAndSet(true)) {
+      // Some other thread is already created for resyncing, do nothing
+    } else {
+      // We have got the lock, create a new thread
+      new Thread() {
+        @Override
+        public void run() {
+          try {
+            if (!rmWorkPreservingRestartEnabled) {
+              LOG.info("Cleaning up running containers on resync");
+              containerManager.cleanupContainersOnNMResync();
+              // Clear all known collectors for resync.
+              if (context.getKnownCollectors() != null) {
+                context.getKnownCollectors().clear();
+              }
+            } else {
+              LOG.info("Preserving containers on resync");
+              // Re-register known timeline collectors.
+              reregisterCollectors();
             }
-          } else {
-            LOG.info("Preserving containers on resync");
-            // Re-register known timeline collectors.
-            reregisterCollectors();
+            ((NodeStatusUpdaterImpl) nodeStatusUpdater)
+                .rebootNodeStatusUpdaterAndRegisterWithRM();
+          } catch (YarnRuntimeException e) {
+            LOG.error("Error while rebooting NodeStatusUpdater.", e);
+            shutDown(NodeManagerStatus.EXCEPTION.getExitCode());
+          } finally {
+            // Release lock
+            resyncingWithRM.set(false);
           }
-          ((NodeStatusUpdaterImpl) nodeStatusUpdater)
-            .rebootNodeStatusUpdaterAndRegisterWithRM();
-        } catch (YarnRuntimeException e) {
-          LOG.error("Error while rebooting NodeStatusUpdater.", e);
-          shutDown(NodeManagerStatus.EXCEPTION.getExitCode());
         }
-      }
-    }.start();
+      }.start();
+    }
   }
 
   /**
@@ -946,7 +957,14 @@ public class NodeManager extends CompositeService
   ContainerManagerImpl getContainerManager() {
     return containerManager;
   }
-  
+
+  /**
+   * Unit test friendly.
+   */
+  protected AsyncDispatcher createNMDispatcher() {
+    return new AsyncDispatcher("NM Event dispatcher");
+  }
+
   //For testing
   Dispatcher getNMDispatcher(){
     return dispatcher;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/10047014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index cf33775..b3f4e1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -37,6 +37,7 @@ import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
@@ -64,7 +65,9 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -107,6 +110,7 @@ public class TestNodeManagerResync {
   private FileContext localFS;
   private CyclicBarrier syncBarrier;
   private CyclicBarrier updateBarrier;
+  private AtomicInteger resyncThreadCount;
   private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
   private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false);
   private final NodeManagerEvent resyncEvent =
@@ -125,6 +129,7 @@ public class TestNodeManagerResync {
     nmLocalDir.mkdirs();
     syncBarrier = new CyclicBarrier(2);
     updateBarrier = new CyclicBarrier(2);
+    resyncThreadCount = new AtomicInteger(0);
   }
 
   @After
@@ -186,6 +191,41 @@ public class TestNodeManagerResync {
   }
 
   @SuppressWarnings("resource")
+  @Test(timeout = 30000)
+  public void testNMMultipleResyncEvent()
+      throws IOException, InterruptedException {
+    TestNodeManager1 nm = new TestNodeManager1(false);
+    YarnConfiguration conf = createNMConfig();
+
+    int resyncEventCount = 4;
+    try {
+      nm.init(conf);
+      nm.start();
+      Assert.assertEquals(1, nm.getNMRegistrationCount());
+      for (int i = 0; i < resyncEventCount; i++) {
+        nm.getNMDispatcher().getEventHandler().handle(resyncEvent);
+      }
+
+      DrainDispatcher dispatcher = (DrainDispatcher) nm.getNMDispatcher();
+      dispatcher.await();
+      LOG.info("NM dispatcher drained");
+
+      // Wait for the resync thread to finish
+      try {
+        syncBarrier.await();
+      } catch (BrokenBarrierException e) {
+      }
+      LOG.info("Barrier wait done for the resync thread");
+
+      // Resync should only happen once
+      Assert.assertEquals(2, nm.getNMRegistrationCount());
+      Assert.assertFalse("NM shutdown called.", isNMShutdownCalled.get());
+    } finally {
+      nm.stop();
+    }
+  }
+
+  @SuppressWarnings("resource")
   @Test(timeout=10000)
   public void testNMshutdownWhenResyncThrowException() throws IOException,
       InterruptedException, YarnException {
@@ -400,6 +440,11 @@ public class TestNodeManagerResync {
     }
 
     @Override
+    protected AsyncDispatcher createNMDispatcher() {
+      return new DrainDispatcher();
+    }
+
+    @Override
     protected NodeStatusUpdater createNodeStatusUpdater(Context context,
         Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
       return new TestNodeStatusUpdaterImpl1(context, dispatcher,
@@ -410,6 +455,14 @@ public class TestNodeManagerResync {
       return registrationCount;
     }
 
+    @Override
+    protected void shutDown(int exitCode) {
+      synchronized (isNMShutdownCalled) {
+        isNMShutdownCalled.set(true);
+        isNMShutdownCalled.notify();
+      }
+    }
+
     class TestNodeStatusUpdaterImpl1 extends MockNodeStatusUpdater {
 
       public TestNodeStatusUpdaterImpl1(Context context, Dispatcher dispatcher,
@@ -428,6 +481,9 @@ public class TestNodeManagerResync {
         ConcurrentMap<ContainerId, org.apache.hadoop.yarn.server.nodemanager
         .containermanager.container.Container> containers =
             getNMContext().getContainers();
+        if (resyncThreadCount.incrementAndGet() > 1) {
+          throw new YarnRuntimeException("Multiple resync thread created!");
+        }
         try {
           try {
             if (containersShouldBePreserved) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[44/50] [abbrv] hadoop git commit: YARN-8455. Add basic ACL check for all ATS v2 REST APIs. Contributed by Rohith Sharma K S.

Posted by gi...@apache.org.
YARN-8455. Add basic ACL check for all ATS v2 REST APIs. Contributed by Rohith Sharma K S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/469b29c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/469b29c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/469b29c0

Branch: refs/heads/HADOOP-15461
Commit: 469b29c0817b7bf1902c9195c4f8d031a909e1c9
Parents: 73746c5
Author: Sunil G <su...@apache.org>
Authored: Fri Jun 29 10:02:53 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Fri Jun 29 10:02:53 2018 -0700

----------------------------------------------------------------------
 .../reader/TimelineFromIdConverter.java         |  93 +++++++++
 .../reader/TimelineReaderWebServices.java       | 198 +++++++++++++++----
 .../TestTimelineReaderWebServicesBasicAcl.java  | 154 +++++++++++++++
 3 files changed, 407 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
new file mode 100644
index 0000000..5f5f0b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.List;
+
+/**
+ * Used for decoding FROM_ID
+ */
+enum TimelineFromIdConverter {
+
+  APPLICATION_FROMID {
+    @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+      if (fromId == null) {
+        return null;
+      }
+
+      List<String> appTupleList = TimelineReaderUtils.split(fromId);
+      if (appTupleList == null || appTupleList.size() != 5) {
+        throw new IllegalArgumentException(
+            "Invalid row key for application table.");
+      }
+
+      return new TimelineReaderContext(appTupleList.get(0), appTupleList.get(1),
+          appTupleList.get(2), Long.parseLong(appTupleList.get(3)),
+          appTupleList.get(4), null, null);
+    }
+  },
+
+  SUB_APPLICATION_ENTITY_FROMID {
+    @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+      if (fromId == null) {
+        return null;
+      }
+      List<String> split = TimelineReaderUtils.split(fromId);
+      if (split == null || split.size() != 6) {
+        throw new IllegalArgumentException(
+            "Invalid row key for sub app table.");
+      }
+
+      String subAppUserId = split.get(0);
+      String clusterId = split.get(1);
+      String entityType = split.get(2);
+      Long entityIdPrefix = Long.valueOf(split.get(3));
+      String entityId = split.get(4);
+      String userId = split.get(5);
+      return new TimelineReaderContext(clusterId, userId, null, null, null,
+          entityType, entityIdPrefix, entityId, subAppUserId);
+    }
+  },
+
+  GENERIC_ENTITY_FROMID {
+    @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+      if (fromId == null) {
+        return null;
+      }
+      List<String> split = TimelineReaderUtils.split(fromId);
+      if (split == null || split.size() != 8) {
+        throw new IllegalArgumentException("Invalid row key for entity table.");
+      }
+      Long flowRunId = Long.valueOf(split.get(3));
+      Long entityIdPrefix = Long.valueOf(split.get(6));
+      return new TimelineReaderContext(split.get(0), split.get(1), split.get(2),
+          flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7));
+    }
+  };
+
+  /**
+   * Decodes FROM_ID depending on FROM_ID implementation.
+   *
+   * @param fromId FROM_ID to be decoded.
+   * @return a {@link TimelineReaderContext} object if FROM_ID passed can be
+   * decoded, null otherwise.
+   * @throws Exception if any problem occurs while decoding.
+   */
+  abstract TimelineReaderContext decodeUID(String fromId) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 7bf66b0..7f96bfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -188,6 +189,8 @@ public class TimelineReaderWebServices {
           "Filter Parsing failed." : e.getMessage());
     } else if (e instanceof BadRequestException) {
       throw (BadRequestException)e;
+    } else if (e instanceof ForbiddenException) {
+      throw (ForbiddenException) e;
     } else {
       LOG.error("Error while processing REST request", e);
       throw new WebApplicationException(e,
@@ -339,6 +342,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForGenericEntities(entities, callerUGI, entityType);
     } catch (Exception e) {
       handleException(e, url, startTime,
           "createdTime start/end or limit or flowrunid");
@@ -607,13 +611,15 @@ public class TimelineReaderWebServices {
           .createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
               appId, entityType, null, null);
       entities = timelineReaderManager.getEntities(context,
-          TimelineReaderWebServicesUtils.createTimelineEntityFilters(
-          limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
-              infofilters, conffilters, metricfilters, eventfilters,
-              fromId),
-          TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
-          confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
-          metricsTimeStart, metricsTimeEnd));
+          TimelineReaderWebServicesUtils
+              .createTimelineEntityFilters(limit, createdTimeStart,
+                  createdTimeEnd, relatesTo, isRelatedTo, infofilters,
+                  conffilters, metricfilters, eventfilters, fromId),
+          TimelineReaderWebServicesUtils
+              .createTimelineDataToRetrieve(confsToRetrieve, metricsToRetrieve,
+                  fields, metricsLimit, metricsTimeStart, metricsTimeEnd));
+
+      checkAccessForGenericEntities(entities, callerUGI, entityType);
     } catch (Exception e) {
       handleException(e, url, startTime,
           "createdTime start/end or limit or flowrunid");
@@ -704,6 +710,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForGenericEntity(entity, callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime, "flowrunid");
     }
@@ -893,6 +900,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForGenericEntity(entity, callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime, "flowrunid");
     }
@@ -956,6 +964,8 @@ public class TimelineReaderWebServices {
       if (context == null) {
         throw new BadRequestException("Incorrect UID " +  uId);
       }
+      // TODO to be removed or modified once ACL story is played
+      checkAccess(timelineReaderManager, callerUGI, context.getUserId());
       context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
       entity = timelineReaderManager.getEntity(context,
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
@@ -1063,12 +1073,16 @@ public class TimelineReaderWebServices {
     TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
     TimelineEntity entity = null;
     try {
-      entity = timelineReaderManager.getEntity(
-          TimelineReaderWebServicesUtils.createTimelineReaderContext(
-          clusterId, userId, flowName, flowRunId, null,
-              TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
-          TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
-          null, metricsToRetrieve, null, null, null, null));
+      TimelineReaderContext context = TimelineReaderWebServicesUtils
+          .createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
+              null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+      // TODO to be removed or modified once ACL story is played
+      checkAccess(timelineReaderManager, callerUGI, context.getUserId());
+
+      entity = timelineReaderManager.getEntity(context,
+          TimelineReaderWebServicesUtils
+              .createTimelineDataToRetrieve(null, metricsToRetrieve, null, null,
+                  null, null));
     } catch (Exception e) {
       handleException(e, url, startTime, "flowrunid");
     }
@@ -1156,6 +1170,8 @@ public class TimelineReaderWebServices {
       if (context == null) {
         throw new BadRequestException("Incorrect UID " +  uId);
       }
+      // TODO to be removed or modified once ACL story is played
+      checkAccess(timelineReaderManager, callerUGI, context.getUserId());
       context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
       entities = timelineReaderManager.getEntities(context,
           TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@@ -1304,15 +1320,21 @@ public class TimelineReaderWebServices {
     TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
     Set<TimelineEntity> entities = null;
     try {
-      entities = timelineReaderManager.getEntities(
-          TimelineReaderWebServicesUtils.createTimelineReaderContext(
-          clusterId, userId, flowName, null, null,
-              TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
-          TimelineReaderWebServicesUtils.createTimelineEntityFilters(
-          limit, createdTimeStart, createdTimeEnd, null, null, null,
-              null, null, null, fromId),
-          TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
-          null, metricsToRetrieve, fields, null, null, null));
+      TimelineReaderContext timelineReaderContext = TimelineReaderWebServicesUtils
+          .createTimelineReaderContext(clusterId, userId, flowName, null,
+              null, TimelineEntityType.YARN_FLOW_RUN.toString(), null,
+              null);
+      // TODO to be removed or modified once ACL story is played
+      checkAccess(timelineReaderManager, callerUGI,
+          timelineReaderContext.getUserId());
+
+      entities = timelineReaderManager.getEntities(timelineReaderContext,
+          TimelineReaderWebServicesUtils
+              .createTimelineEntityFilters(limit, createdTimeStart,
+                  createdTimeEnd, null, null, null, null, null, null, fromId),
+          TimelineReaderWebServicesUtils
+              .createTimelineDataToRetrieve(null, metricsToRetrieve, fields,
+                  null, null, null));
     } catch (Exception e) {
       handleException(e, url, startTime,
           "createdTime start/end or limit or fromId");
@@ -1435,7 +1457,6 @@ public class TimelineReaderWebServices {
     long startTime = Time.monotonicNow();
     init(res);
     TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
-    Configuration config = timelineReaderManager.getConfig();
     Set<TimelineEntity> entities = null;
     try {
       DateRange range = parseDateRange(dateRange);
@@ -1455,19 +1476,9 @@ public class TimelineReaderWebServices {
     long endTime = Time.monotonicNow();
     if (entities == null) {
       entities = Collections.emptySet();
-    } else if (isDisplayEntityPerUserFilterEnabled(config)) {
-      Set<TimelineEntity> userEntities = new LinkedHashSet<>();
-      userEntities.addAll(entities);
-      for (TimelineEntity entity : userEntities) {
-        if (entity.getInfo() != null) {
-          String userId =
-              (String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
-          if (!validateAuthUserWithEntityUser(timelineReaderManager, callerUGI,
-              userId)) {
-            entities.remove(entity);
-          }
-        }
-      }
+    } else {
+      checkAccess(timelineReaderManager, callerUGI, entities,
+          FlowActivityEntity.USER_INFO_KEY, true);
     }
     LOG.info("Processed URL " + url +
         " (Took " + (endTime - startTime) + " ms.)");
@@ -1552,6 +1563,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForAppEntity(entity, callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime, "flowrunid");
     }
@@ -1722,6 +1734,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForAppEntity(entity, callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime, "flowrunid");
     }
@@ -1852,6 +1865,8 @@ public class TimelineReaderWebServices {
       if (context == null) {
         throw new BadRequestException("Incorrect UID " +  uId);
       }
+      // TODO to be removed or modified once ACL story is played
+      checkAccess(timelineReaderManager, callerUGI, context.getUserId());
       context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
       entities = timelineReaderManager.getEntities(context,
           TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@@ -3343,6 +3358,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
           confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
           metricsTimeStart, metricsTimeEnd));
+      checkAccessForSubAppEntities(entities,callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime,
           "createdTime start/end or limit");
@@ -3410,6 +3426,7 @@ public class TimelineReaderWebServices {
           TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
               confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
               metricsTimeStart, metricsTimeEnd));
+      checkAccessForSubAppEntities(entities,callerUGI);
     } catch (Exception e) {
       handleException(e, url, startTime, "");
     }
@@ -3422,7 +3439,7 @@ public class TimelineReaderWebServices {
     return entities;
   }
 
-  private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
+  static boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
     return !config
         .getBoolean(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED,
             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED)
@@ -3430,8 +3447,76 @@ public class TimelineReaderWebServices {
         .getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
   }
 
+  // TODO to be removed or modified once ACL story is played
+  private void checkAccessForSubAppEntities(Set<TimelineEntity> entities,
+      UserGroupInformation callerUGI) throws Exception {
+    if (entities != null && entities.size() > 0
+        && isDisplayEntityPerUserFilterEnabled(
+        getTimelineReaderManager().getConfig())) {
+      TimelineReaderContext timelineReaderContext = null;
+      TimelineEntity entity = entities.iterator().next();
+      String fromId =
+          (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+      timelineReaderContext =
+          TimelineFromIdConverter.SUB_APPLICATION_ENTITY_FROMID
+              .decodeUID(fromId);
+      checkAccess(getTimelineReaderManager(), callerUGI,
+          timelineReaderContext.getDoAsUser());
+    }
+  }
+
+  // TODO to be removed or modified once ACL story is played
+  private void checkAccessForAppEntity(TimelineEntity entity,
+      UserGroupInformation callerUGI) throws Exception {
+    if (entity != null && isDisplayEntityPerUserFilterEnabled(
+        getTimelineReaderManager().getConfig())) {
+      String fromId =
+          (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+      TimelineReaderContext timelineReaderContext =
+          TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(fromId);
+      checkAccess(getTimelineReaderManager(), callerUGI,
+          timelineReaderContext.getUserId());
+    }
+  }
+
+  // TODO to be removed or modified once ACL story is played
+  private void checkAccessForGenericEntity(TimelineEntity entity,
+      UserGroupInformation callerUGI) throws Exception {
+    if (entity != null && isDisplayEntityPerUserFilterEnabled(
+        getTimelineReaderManager().getConfig())) {
+      String fromId =
+          (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+      TimelineReaderContext timelineReaderContext =
+          TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(fromId);
+      checkAccess(getTimelineReaderManager(), callerUGI,
+          timelineReaderContext.getUserId());
+    }
+  }
+
+  // TODO to be removed or modified once ACL story is played
+  private void checkAccessForGenericEntities(Set<TimelineEntity> entities,
+      UserGroupInformation callerUGI, String entityType) throws Exception {
+    if (entities != null && entities.size() > 0
+        && isDisplayEntityPerUserFilterEnabled(
+        getTimelineReaderManager().getConfig())) {
+      TimelineReaderContext timelineReaderContext = null;
+      TimelineEntity entity = entities.iterator().next();
+      String uid =
+          (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+      if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+        timelineReaderContext =
+            TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(uid);
+      } else {
+        timelineReaderContext =
+            TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(uid);
+      }
+      checkAccess(getTimelineReaderManager(), callerUGI,
+          timelineReaderContext.getUserId());
+    }
+  }
+
   // TODO to be removed/modified once ACL story has played
-  private boolean validateAuthUserWithEntityUser(
+  static boolean validateAuthUserWithEntityUser(
       TimelineReaderManager readerManager, UserGroupInformation ugi,
       String entityUser) {
     String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
@@ -3442,4 +3527,41 @@ public class TimelineReaderWebServices {
     }
     return (readerManager.checkAccess(ugi) || authUser.equals(requestedUser));
   }
+
+  // TODO to be removed/modified once ACL story has played
+  static boolean checkAccess(TimelineReaderManager readerManager,
+      UserGroupInformation ugi, String entityUser) {
+    if (isDisplayEntityPerUserFilterEnabled(readerManager.getConfig())) {
+      if (!validateAuthUserWithEntityUser(readerManager, ugi, entityUser)) {
+        String userName = ugi.getShortUserName();
+        String msg = "User " + userName
+            + " is not allowed to read TimelineService V2 data.";
+        throw new ForbiddenException(msg);
+      }
+    }
+    return true;
+  }
+
+  // TODO to be removed or modified once ACL story is played
+  static void checkAccess(TimelineReaderManager readerManager,
+      UserGroupInformation callerUGI, Set<TimelineEntity> entities,
+      String entityUserKey, boolean verifyForAllEntity) {
+    if (entities.size() > 0 && isDisplayEntityPerUserFilterEnabled(
+        readerManager.getConfig())) {
+      Set<TimelineEntity> userEntities = new LinkedHashSet<>();
+      userEntities.addAll(entities);
+      for (TimelineEntity entity : userEntities) {
+        if (entity.getInfo() != null) {
+          String userId = (String) entity.getInfo().get(entityUserKey);
+          if (!validateAuthUserWithEntityUser(readerManager, callerUGI,
+              userId)) {
+            entities.remove(entity);
+            if (!verifyForAllEntity) {
+              break;
+            }
+          }
+        }
+      }
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
new file mode 100644
index 0000000..4239bf0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class TestTimelineReaderWebServicesBasicAcl {
+
+  private TimelineReaderManager manager;
+  private static String adminUser = "admin";
+  private static UserGroupInformation adminUgi =
+      UserGroupInformation.createRemoteUser(adminUser);
+  private Configuration config;
+
+  @Before public void setUp() throws Exception {
+    config = new YarnConfiguration();
+  }
+
+  @After public void tearDown() throws Exception {
+    if (manager != null) {
+      manager.stop();
+      manager = null;
+    }
+    config = null;
+  }
+
+  @Test public void testTimelineReaderManagerAclsWhenDisabled()
+      throws Exception {
+    config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
+    config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
+    manager = new TimelineReaderManager(null);
+    manager.init(config);
+    manager.start();
+
+    // when acls are disabled, always return true
+    Assert.assertTrue(manager.checkAccess(null));
+
+    // filter is disabled, so should return false
+    Assert.assertFalse(
+        TimelineReaderWebServices.isDisplayEntityPerUserFilterEnabled(config));
+  }
+
+  @Test public void testTimelineReaderManagerAclsWhenEnabled()
+      throws Exception {
+    Configuration config = new YarnConfiguration();
+    config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+    config.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
+    config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
+    manager = new TimelineReaderManager(null);
+    manager.init(config);
+    manager.start();
+
+    String user1 = "user1";
+    String user2 = "user2";
+    UserGroupInformation user1Ugi =
+        UserGroupInformation.createRemoteUser(user1);
+    UserGroupInformation user2Ugi =
+        UserGroupInformation.createRemoteUser(user2);
+
+    // false because ugi is null
+    Assert.assertFalse(TimelineReaderWebServices
+        .validateAuthUserWithEntityUser(manager, null, user1));
+
+    // incoming ugi is admin asking for entity owner user1
+    Assert.assertTrue(
+        TimelineReaderWebServices.checkAccess(manager, adminUgi, user1));
+
+    // incoming ugi is admin asking for entity owner user1
+    Assert.assertTrue(
+        TimelineReaderWebServices.checkAccess(manager, adminUgi, user2));
+
+    // incoming ugi is non-admin i.e user1Ugi asking for entity owner user2
+    try {
+      TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
+      Assert.fail("user1Ugi is not allowed to view user2");
+    } catch (ForbiddenException e) {
+      // expected
+    }
+
+    // incoming ugi is non-admin i.e user2Ugi asking for entity owner user1
+    try {
+      TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
+      Assert.fail("user2Ugi is not allowed to view user1");
+    } catch (ForbiddenException e) {
+      // expected
+    }
+
+    String userKey = "user";
+    // incoming ugi is admin asking for entities
+    Set<TimelineEntity> entities = createEntities(10, userKey);
+    TimelineReaderWebServices
+        .checkAccess(manager, adminUgi, entities, userKey, true);
+    // admin is allowed to view other entities
+    Assert.assertTrue(entities.size() == 10);
+
+    // incoming ugi is user1Ugi asking for entities
+    // only user1 entities are allowed to view
+    entities = createEntities(5, userKey);
+    TimelineReaderWebServices
+        .checkAccess(manager, user1Ugi, entities, userKey, true);
+    Assert.assertTrue(entities.size() == 1);
+    Assert
+        .assertEquals(user1, entities.iterator().next().getInfo().get(userKey));
+
+    // incoming ugi is user2Ugi asking for entities
+    // only user2 entities are allowed to view
+    entities = createEntities(8, userKey);
+    TimelineReaderWebServices
+        .checkAccess(manager, user2Ugi, entities, userKey, true);
+    Assert.assertTrue(entities.size() == 1);
+    Assert
+        .assertEquals(user2, entities.iterator().next().getInfo().get(userKey));
+  }
+
+  Set<TimelineEntity> createEntities(int noOfUsers, String userKey) {
+    Set<TimelineEntity> entities = new LinkedHashSet<>();
+    for (int i = 0; i < noOfUsers; i++) {
+      TimelineEntity e = new TimelineEntity();
+      e.setType("user" + i);
+      e.setId("user" + i);
+      e.getInfo().put(userKey, "user" + i);
+      entities.add(e);
+    }
+    return entities;
+  }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[11/50] [abbrv] hadoop git commit: HADOOP-14396. Add builder interface to FileContext. Contributed by Lei (Eddy) Xu.

Posted by gi...@apache.org.
HADOOP-14396. Add builder interface to FileContext.
Contributed by  Lei (Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba4e623
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba4e623
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba4e623

Branch: refs/heads/HADOOP-15461
Commit: 1ba4e62304a70d53f1a4f76995b6e1fac3107922
Parents: 440140c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 25 14:38:33 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 25 14:38:57 2018 +0100

----------------------------------------------------------------------
 .../hadoop/fs/FSDataOutputStreamBuilder.java    | 22 +++++++
 .../java/org/apache/hadoop/fs/FileContext.java  | 66 ++++++++++++++++++++
 .../main/java/org/apache/hadoop/fs/Options.java |  3 +
 .../fs/FileContextMainOperationsBaseTest.java   | 44 ++++++++++++-
 4 files changed, 134 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 86c284a..d431293 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -116,6 +116,27 @@ public abstract class FSDataOutputStreamBuilder
   protected abstract B getThisBuilder();
 
   /**
+   * Construct from a {@link FileContext}.
+   *
+   * @param fc FileContext
+   * @param p path.
+   * @throws IOException
+   */
+  FSDataOutputStreamBuilder(@Nonnull FileContext fc,
+      @Nonnull Path p) throws IOException {
+    Preconditions.checkNotNull(fc);
+    Preconditions.checkNotNull(p);
+    this.fs = null;
+    this.path = p;
+
+    AbstractFileSystem afs = fc.getFSofPath(p);
+    FsServerDefaults defaults = afs.getServerDefaults(p);
+    bufferSize = defaults.getFileBufferSize();
+    replication = defaults.getReplication();
+    blockSize = defaults.getBlockSize();
+  }
+
+  /**
    * Constructor.
    */
   protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
@@ -131,6 +152,7 @@ public abstract class FSDataOutputStreamBuilder
   }
 
   protected FileSystem getFS() {
+    Preconditions.checkNotNull(fs);
     return fs;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 6ea69d0..5215c3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
 import java.net.URI;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashSet;
@@ -35,6 +36,8 @@ import java.util.Stack;
 import java.util.TreeSet;
 import java.util.Map.Entry;
 
+import javax.annotation.Nonnull;
+
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -695,6 +698,69 @@ public class FileContext {
   }
 
   /**
+   * {@link FSDataOutputStreamBuilder} for {@liink FileContext}.
+   */
+  private static final class FCDataOutputStreamBuilder extends
+      FSDataOutputStreamBuilder<
+        FSDataOutputStream, FCDataOutputStreamBuilder> {
+    private final FileContext fc;
+
+    private FCDataOutputStreamBuilder(
+        @Nonnull FileContext fc, @Nonnull Path p) throws IOException {
+      super(fc, p);
+      this.fc = fc;
+      Preconditions.checkNotNull(fc);
+    }
+
+    @Override
+    protected FCDataOutputStreamBuilder getThisBuilder() {
+      return this;
+    }
+
+    @Override
+    public FSDataOutputStream build() throws IOException {
+      final EnumSet<CreateFlag> flags = getFlags();
+      List<CreateOpts> createOpts = new ArrayList<>(Arrays.asList(
+          CreateOpts.blockSize(getBlockSize()),
+          CreateOpts.bufferSize(getBufferSize()),
+          CreateOpts.repFac(getReplication()),
+          CreateOpts.perms(getPermission())
+      ));
+      if (getChecksumOpt() != null) {
+        createOpts.add(CreateOpts.checksumParam(getChecksumOpt()));
+      }
+      if (getProgress() != null) {
+        createOpts.add(CreateOpts.progress(getProgress()));
+      }
+      if (isRecursive()) {
+        createOpts.add(CreateOpts.createParent());
+      }
+      return fc.create(getPath(), flags,
+          createOpts.toArray(new CreateOpts[0]));
+    }
+  }
+
+  /**
+   * Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+   * a file on indicated path.
+   *
+   * @param f the file path to create builder for.
+   * @return {@link FSDataOutputStreamBuilder} to build a
+   *         {@link FSDataOutputStream}.
+   *
+   * Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+   * builder parameters will be verified by {@link FileContext} and
+   * {@link AbstractFileSystem#create}. And filesystem states will be modified.
+   *
+   * Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+   * same exceptions as create(Path, EnumSet, CreateOpts...).
+   */
+  public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
+      throws IOException {
+    return new FCDataOutputStreamBuilder(this, f).create();
+  }
+
+  /**
    * Make(create) a directory and all the non-existent parents.
    * 
    * @param dir - the dir to make

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 126e754..5e93286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -55,6 +55,9 @@ public final class Options {
         ChecksumOpt csumOpt) {
       return new ChecksumParam(csumOpt);
     }
+    public static Progress progress(Progressable prog) {
+      return new Progress(prog);
+    }
     public static Perms perms(FsPermission perm) {
       return new Perms(perm);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 35ec4ff..62ecd9f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -810,7 +810,49 @@ public abstract class FileContextMainOperationsBaseTest  {
     fc.create(p, EnumSet.of(CREATE, APPEND, OVERWRITE));
     Assert.fail("Excepted exception not thrown");
   }
-  
+
+  @Test
+  public void testBuilderCreateNonExistingFile() throws IOException {
+    Path p = getTestRootPath(fc, "test/testBuilderCreateNonExistingFile");
+    FSDataOutputStream out = fc.create(p).build();
+    writeData(fc, p, out, data, data.length);
+  }
+
+  @Test
+  public void testBuilderCreateExistingFile() throws IOException {
+    Path p = getTestRootPath(fc, "test/testBuilderCreateExistingFile");
+    createFile(p);
+    FSDataOutputStream out = fc.create(p).overwrite(true).build();
+    writeData(fc, p, out, data, data.length);
+  }
+
+  @Test
+  public void testBuilderCreateAppendNonExistingFile() throws IOException {
+    Path p = getTestRootPath(fc, "test/testBuilderCreateAppendNonExistingFile");
+    FSDataOutputStream out = fc.create(p).append().build();
+    writeData(fc, p, out, data, data.length);
+  }
+
+  @Test
+  public void testBuilderCreateAppendExistingFile() throws IOException {
+    Path p = getTestRootPath(fc, "test/testBuilderCreateAppendExistingFile");
+    createFile(p);
+    FSDataOutputStream out = fc.create(p).append().build();
+    writeData(fc, p, out, data, 2 * data.length);
+  }
+
+  @Test
+  public void testBuilderCreateRecursive() throws IOException {
+    Path p = getTestRootPath(fc, "test/parent/no/exist/file1");
+    try (FSDataOutputStream out = fc.create(p).build()) {
+      fail("Should throw FileNotFoundException on non-exist directory");
+    } catch (FileNotFoundException e) {
+    }
+
+    FSDataOutputStream out = fc.create(p).recursive().build();
+    writeData(fc, p, out, data, data.length);
+  }
+
   private static void writeData(FileContext fc, Path p, FSDataOutputStream out,
       byte[] data, long expectedLen) throws IOException {
     out.write(data, 0, data.length);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[21/50] [abbrv] hadoop git commit: YARN-8461. Support strict memory control on individual container with elastic control memory mechanism. Contributed by Haibo Chen.

Posted by gi...@apache.org.
YARN-8461. Support strict memory control on individual container with elastic control memory mechanism. Contributed by Haibo Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62d83ca5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62d83ca5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62d83ca5

Branch: refs/heads/HADOOP-15461
Commit: 62d83ca5360cf803ecf6780caf192462d0092009
Parents: b69ba0f
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Tue Jun 26 15:21:35 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Tue Jun 26 15:21:35 2018 -0700

----------------------------------------------------------------------
 .../CGroupsMemoryResourceHandlerImpl.java       |  24 +++++
 .../linux/resources/MemoryResourceHandler.java  |  10 ++
 .../monitor/ContainersMonitorImpl.java          | 108 +++++++++++--------
 .../TestCGroupsMemoryResourceHandlerImpl.java   |  43 ++++++++
 4 files changed, 142 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index a57adb1..053b796 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -34,6 +34,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
 
 /**
  * Handler class to handle the memory controller. YARN already ships a
@@ -172,4 +175,25 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
     return null;
   }
 
+  @Override
+  public Optional<Boolean> isUnderOOM(ContainerId containerId) {
+    try {
+      String status = cGroupsHandler.getCGroupParam(
+          CGroupsHandler.CGroupController.MEMORY,
+          containerId.toString(),
+          CGROUP_PARAM_MEMORY_OOM_CONTROL);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("cgroups OOM status for " + containerId + ": " + status);
+      }
+      if (status.contains(CGroupsHandler.UNDER_OOM)) {
+        LOG.warn("Container " + containerId + " under OOM based on cgroups.");
+        return Optional.of(true);
+      } else {
+        return Optional.of(false);
+      }
+    } catch (ResourceHandlerException e) {
+      LOG.warn("Could not read cgroups" + containerId, e);
+    }
+    return Optional.empty();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
index 013a49f..1729fc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
@@ -20,8 +20,18 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+import java.util.Optional;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public interface MemoryResourceHandler extends ResourceHandler {
+  /**
+   * check whether a container is under OOM.
+   * @param containerId the id of the container
+   * @return empty if the status is unknown, true is the container is under oom,
+   *         false otherwise
+   */
+  Optional<Boolean> isUnderOOM(ContainerId containerId);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index bd68dfe..d83fe39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.slf4j.Logger;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
@@ -697,55 +699,75 @@ public class ContainersMonitorImpl extends AbstractService implements
                             ProcessTreeInfo ptInfo,
                             long currentVmemUsage,
                             long currentPmemUsage) {
-      if (elasticMemoryEnforcement || strictMemoryEnforcement) {
-        // We enforce the overall memory usage instead of individual containers
-        return;
-      }
-      boolean isMemoryOverLimit = false;
-      long vmemLimit = ptInfo.getVmemLimit();
-      long pmemLimit = ptInfo.getPmemLimit();
-      // as processes begin with an age 1, we want to see if there
-      // are processes more than 1 iteration old.
-      long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
-      long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
+      Optional<Boolean> isMemoryOverLimit = Optional.empty();
       String msg = "";
       int containerExitStatus = ContainerExitStatus.INVALID;
-      if (isVmemCheckEnabled()
-              && isProcessTreeOverLimit(containerId.toString(),
-              currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
-        // The current usage (age=0) is always higher than the aged usage. We
-        // do not show the aged size in the message, base the delta on the
-        // current usage
-        long delta = currentVmemUsage - vmemLimit;
-        // Container (the root process) is still alive and overflowing
-        // memory.
-        // Dump the process-tree and then clean it up.
-        msg = formatErrorMessage("virtual",
-                formatUsageString(currentVmemUsage, vmemLimit,
+
+      if (strictMemoryEnforcement && elasticMemoryEnforcement) {
+        // Both elastic memory control and strict memory control are enabled
+        // through cgroups. A container will be frozen by the elastic memory
+        // control mechanism if it exceeds its request, so we check for this
+        // here and kill it. Otherwise, the container will not be killed if
+        // the node never exceeds its limit and the procfs-based
+        // memory accounting is different from the cgroup-based accounting.
+
+        MemoryResourceHandler handler =
+            ResourceHandlerModule.getMemoryResourceHandler();
+        if (handler != null) {
+          isMemoryOverLimit = handler.isUnderOOM(containerId);
+          containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+          msg = containerId + " is under oom because it exceeded its" +
+              " physical memory limit";
+        }
+      } else if (strictMemoryEnforcement || elasticMemoryEnforcement) {
+        // if cgroup-based memory control is enabled
+        isMemoryOverLimit = Optional.of(false);
+      }
+
+      if (!isMemoryOverLimit.isPresent()) {
+        long vmemLimit = ptInfo.getVmemLimit();
+        long pmemLimit = ptInfo.getPmemLimit();
+        // as processes begin with an age 1, we want to see if there
+        // are processes more than 1 iteration old.
+        long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
+        long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
+        if (isVmemCheckEnabled()
+            && isProcessTreeOverLimit(containerId.toString(),
+            currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
+          // The current usage (age=0) is always higher than the aged usage. We
+          // do not show the aged size in the message, base the delta on the
+          // current usage
+          long delta = currentVmemUsage - vmemLimit;
+          // Container (the root process) is still alive and overflowing
+          // memory.
+          // Dump the process-tree and then clean it up.
+          msg = formatErrorMessage("virtual",
+              formatUsageString(currentVmemUsage, vmemLimit,
                   currentPmemUsage, pmemLimit),
-                pId, containerId, pTree, delta);
-        isMemoryOverLimit = true;
-        containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM;
-      } else if (isPmemCheckEnabled()
-              && isProcessTreeOverLimit(containerId.toString(),
-              currentPmemUsage, curRssMemUsageOfAgedProcesses,
-              pmemLimit)) {
-        // The current usage (age=0) is always higher than the aged usage. We
-        // do not show the aged size in the message, base the delta on the
-        // current usage
-        long delta = currentPmemUsage - pmemLimit;
-        // Container (the root process) is still alive and overflowing
-        // memory.
-        // Dump the process-tree and then clean it up.
-        msg = formatErrorMessage("physical",
-                formatUsageString(currentVmemUsage, vmemLimit,
+              pId, containerId, pTree, delta);
+          isMemoryOverLimit = Optional.of(true);
+          containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM;
+        } else if (isPmemCheckEnabled()
+            && isProcessTreeOverLimit(containerId.toString(),
+            currentPmemUsage, curRssMemUsageOfAgedProcesses,
+            pmemLimit)) {
+          // The current usage (age=0) is always higher than the aged usage. We
+          // do not show the aged size in the message, base the delta on the
+          // current usage
+          long delta = currentPmemUsage - pmemLimit;
+          // Container (the root process) is still alive and overflowing
+          // memory.
+          // Dump the process-tree and then clean it up.
+          msg = formatErrorMessage("physical",
+              formatUsageString(currentVmemUsage, vmemLimit,
                   currentPmemUsage, pmemLimit),
-                pId, containerId, pTree, delta);
-        isMemoryOverLimit = true;
-        containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+              pId, containerId, pTree, delta);
+          isMemoryOverLimit = Optional.of(true);
+          containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+        }
       }
 
-      if (isMemoryOverLimit) {
+      if (isMemoryOverLimit.isPresent() && isMemoryOverLimit.get()) {
         // Virtual or physical memory over limit. Fail the container and
         // remove
         // the corresponding process tree

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
index 5c7e233..4d3e7e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
@@ -31,7 +31,9 @@ import org.junit.Test;
 import org.junit.Assert;
 
 import java.util.List;
+import java.util.Optional;
 
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
 import static org.mockito.Mockito.*;
 
 /**
@@ -242,4 +244,45 @@ public class TestCGroupsMemoryResourceHandlerImpl {
         .updateCGroupParam(CGroupsHandler.CGroupController.MEMORY, id,
             CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES, "1024M");
   }
+
+  @Test
+  public void testContainerUnderOom() throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+
+    cGroupsMemoryResourceHandler.bootstrap(conf);
+
+    ContainerId containerId = mock(ContainerId.class);
+    when(containerId.toString()).thenReturn("container_01_01");
+
+    when(mockCGroupsHandler.getCGroupParam(
+        CGroupsHandler.CGroupController.MEMORY,
+        containerId.toString(),
+        CGROUP_PARAM_MEMORY_OOM_CONTROL)).thenReturn(CGroupsHandler.UNDER_OOM);
+    Optional<Boolean> outOfOom =
+        cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+    Assert.assertTrue("The container should be reported to run under oom",
+        outOfOom.isPresent() && outOfOom.get().equals(true));
+
+    when(mockCGroupsHandler.getCGroupParam(
+        CGroupsHandler.CGroupController.MEMORY,
+        containerId.toString(),
+        CGROUP_PARAM_MEMORY_OOM_CONTROL)).thenReturn("");
+    outOfOom = cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+    Assert.assertTrue(
+        "The container should not be reported to run under oom",
+        outOfOom.isPresent() && outOfOom.get().equals(false));
+
+    when(mockCGroupsHandler.getCGroupParam(
+        CGroupsHandler.CGroupController.MEMORY,
+        containerId.toString(),
+        CGROUP_PARAM_MEMORY_OOM_CONTROL)).
+        thenThrow(new ResourceHandlerException());
+    outOfOom = cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+    Assert.assertFalse(
+        "No report of the oom status should be available.",
+        outOfOom.isPresent());
+
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[47/50] [abbrv] hadoop git commit: HADOOP-15548: Randomize local dirs. Contributed by Jim Brennan.

Posted by gi...@apache.org.
HADOOP-15548: Randomize local dirs. Contributed by Jim Brennan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d36f6b9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d36f6b9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d36f6b9e

Branch: refs/heads/HADOOP-15461
Commit: d36f6b9e93e4c30d24d0e837cb00bd24ffa8f274
Parents: 1004701
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 29 18:18:32 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 29 18:18:32 2018 +0000

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/LocalDirAllocator.java |  7 ++-
 .../apache/hadoop/fs/TestLocalDirAllocator.java | 59 ++++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36f6b9e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index c1e9d21..1c216f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -418,7 +418,12 @@ public class LocalDirAllocator {
           }
         }
       } else {
-        int dirNum = ctx.getAndIncrDirNumLastAccessed();
+        // Start linear search with random increment if possible
+        int randomInc = 1;
+        if (numDirs > 2) {
+          randomInc += dirIndexRandomizer.nextInt(numDirs - 1);
+        }
+        int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc);
         while (numDirsSearched < numDirs) {
           long capacity = ctx.dirDF[dirNum].getAvailable();
           if (capacity > size) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36f6b9e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
index 825efe0..acda898 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
@@ -265,6 +265,65 @@ public class TestLocalDirAllocator {
     }
   }
 
+  /**
+   * Five buffer dirs, on read-write disk.
+   *
+   * Try to create a whole bunch of files.
+   *  Verify that each successive creation uses a different disk
+   *  than the previous one (for sized requests).
+   *
+   *  Would ideally check statistical properties of distribution, but
+   *  we don't have the nerve to risk false-positives here.
+   *
+   * @throws Exception
+   */
+  @Test (timeout = 30000)
+  public void testCreateManyFilesRandom() throws Exception {
+    assumeNotWindows();
+    final int numDirs = 5;
+    final int numTries = 100;
+    String[] dirs = new String[numDirs];
+    for (int d = 0; d < numDirs; ++d) {
+      dirs[d] = buildBufferDir(ROOT, d);
+    }
+    boolean next_dir_not_selected_at_least_once = false;
+    try {
+      conf.set(CONTEXT, dirs[0] + "," + dirs[1] + "," + dirs[2] + ","
+          + dirs[3] + "," + dirs[4]);
+      Path[] paths = new Path[5];
+      for (int d = 0; d < numDirs; ++d) {
+        paths[d] = new Path(dirs[d]);
+        assertTrue(localFs.mkdirs(paths[d]));
+      }
+
+      int inDir=0;
+      int prevDir = -1;
+      int[] counts = new int[5];
+      for(int i = 0; i < numTries; ++i) {
+        File result = createTempFile(SMALL_FILE_SIZE);
+        for (int d = 0; d < numDirs; ++d) {
+          if (result.getPath().startsWith(paths[d].toUri().getPath())) {
+            inDir = d;
+            break;
+          }
+        }
+        // Verify we always select a different dir
+        assertNotEquals(prevDir, inDir);
+        // Verify we are not always selecting the next dir - that was the old
+        // algorithm.
+        if ((prevDir != -1) && (inDir != ((prevDir + 1) % numDirs))) {
+          next_dir_not_selected_at_least_once = true;
+        }
+        prevDir = inDir;
+        counts[inDir]++;
+        result.delete();
+      }
+    } finally {
+      rmBufferDirs();
+    }
+    assertTrue(next_dir_not_selected_at_least_once);
+  }
+
   /** Two buffer dirs. The first dir does not exist & is on a read-only disk;
    * The second dir exists & is RW
    * getLocalPathForWrite with checkAccess set to false should create a parent


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[09/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable ozonefs artifact Contributed by Marton, Elek.

Posted by gi...@apache.org.
HDDS-177. Create a releasable ozonefs artifact
Contributed by Marton, Elek.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e16e5b30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e16e5b30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e16e5b30

Branch: refs/heads/HADOOP-15461
Commit: e16e5b307d6c4404db0698b9d128e5bf4aa16a8a
Parents: ca14fec
Author: Anu Engineer <ae...@apache.org>
Authored: Sun Jun 24 01:05:04 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sun Jun 24 01:05:04 2018 -0700

----------------------------------------------------------------------
 dev-support/bin/ozone-dist-layout-stitching     |   4 +-
 hadoop-dist/pom.xml                             |   5 +
 .../test/acceptance/ozonefs/docker-compose.yaml |  71 ++
 .../src/test/acceptance/ozonefs/docker-config   |  39 ++
 .../src/test/acceptance/ozonefs/ozonefs.robot   |  39 ++
 hadoop-ozone/ozonefs/pom.xml                    | 211 ++++++
 .../org/apache/hadoop/fs/ozone/Constants.java   |  42 ++
 .../java/org/apache/hadoop/fs/ozone/OzFs.java   |  44 ++
 .../hadoop/fs/ozone/OzoneFSInputStream.java     |  79 +++
 .../hadoop/fs/ozone/OzoneFSOutputStream.java    |  59 ++
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
 .../apache/hadoop/fs/ozone/package-info.java    |  30 +
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
 .../fs/ozone/TestOzoneFileInterfaces.java       | 231 +++++++
 .../contract/ITestOzoneContractCreate.java      |  48 ++
 .../contract/ITestOzoneContractDelete.java      |  48 ++
 .../contract/ITestOzoneContractDistCp.java      |  50 ++
 .../ITestOzoneContractGetFileStatus.java        |  61 ++
 .../ozone/contract/ITestOzoneContractMkdir.java |  48 ++
 .../ozone/contract/ITestOzoneContractOpen.java  |  47 ++
 .../contract/ITestOzoneContractRename.java      |  49 ++
 .../contract/ITestOzoneContractRootDir.java     |  51 ++
 .../ozone/contract/ITestOzoneContractSeek.java  |  47 ++
 .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
 .../src/test/resources/contract/ozone.xml       | 113 +++
 .../ozonefs/src/test/resources/log4j.properties |  23 +
 hadoop-ozone/pom.xml                            |   1 +
 hadoop-project/pom.xml                          |   6 +-
 hadoop-tools/hadoop-ozone/pom.xml               | 174 -----
 .../org/apache/hadoop/fs/ozone/Constants.java   |  42 --
 .../java/org/apache/hadoop/fs/ozone/OzFs.java   |  44 --
 .../hadoop/fs/ozone/OzoneFSInputStream.java     |  79 ---
 .../hadoop/fs/ozone/OzoneFSOutputStream.java    |  59 --
 .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
 .../apache/hadoop/fs/ozone/package-info.java    |  30 -
 .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
 .../fs/ozone/TestOzoneFileInterfaces.java       | 231 -------
 .../contract/ITestOzoneContractCreate.java      |  48 --
 .../contract/ITestOzoneContractDelete.java      |  48 --
 .../contract/ITestOzoneContractDistCp.java      |  50 --
 .../ITestOzoneContractGetFileStatus.java        |  61 --
 .../ozone/contract/ITestOzoneContractMkdir.java |  48 --
 .../ozone/contract/ITestOzoneContractOpen.java  |  47 --
 .../contract/ITestOzoneContractRename.java      |  49 --
 .../contract/ITestOzoneContractRootDir.java     |  51 --
 .../ozone/contract/ITestOzoneContractSeek.java  |  47 --
 .../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
 .../src/test/resources/contract/ozone.xml       | 113 ---
 .../src/test/resources/log4j.properties         |  23 -
 hadoop-tools/hadoop-tools-dist/pom.xml          |  15 -
 hadoop-tools/pom.xml                            |  11 -
 51 files changed, 2413 insertions(+), 2241 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index ad8abe2..be330d5 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -145,6 +145,8 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
 run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
 run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
 run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+mkdir -p "./share/hadoop/ozonefs"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
 # Optional documentation, could be missing
 cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
 cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
@@ -153,5 +155,5 @@ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdd
 mkdir -p ./share/hadoop/mapreduce
 mkdir -p ./share/hadoop/yarn
 echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
+echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
 echo

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index dfbf818..5de6759 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -265,6 +265,11 @@
           <artifactId>hadoop-ozone-docs</artifactId>
           <scope>provided</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-ozone-filesystem</artifactId>
+          <scope>provided</scope>
+        </dependency>
       </dependencies>
       <build>
         <plugins>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
new file mode 100644
index 0000000..3323557
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+   namenode:
+      image: apache/hadoop-runner
+      hostname: namenode
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9870
+      environment:
+          ENSURE_NAMENODE_DIR: /data/namenode
+      env_file:
+         - ./docker-config
+      command: ["/opt/hadoop/bin/hdfs","namenode"]
+   datanode:
+      image: apache/hadoop-runner
+      volumes:
+        - ${OZONEDIR}:/opt/hadoop
+      ports:
+        - 9864
+      command: ["/opt/hadoop/bin/ozone","datanode"]
+      env_file:
+        - ./docker-config
+   ksm:
+      image: apache/hadoop-runner
+      hostname: ksm
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9874
+      environment:
+         ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+      env_file:
+          - ./docker-config
+      command: ["/opt/hadoop/bin/ozone","ksm"]
+   scm:
+      image: apache/hadoop-runner
+      volumes:
+         - ${OZONEDIR}:/opt/hadoop
+      ports:
+         - 9876
+      env_file:
+          - ./docker-config
+      environment:
+          ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+      command: ["/opt/hadoop/bin/ozone","scm"]
+   hadooplast:
+      image: flokkr/hadoop:3.1.0
+      volumes:
+        - ${OZONEDIR}:/opt/ozone
+      env_file:
+        - ./docker-config
+      environment:
+         HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+      command: ["watch","-n","100000","ls"]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
new file mode 100644
index 0000000..dec863e
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.category.org.apache.hadoop.util.NativeCodeLoader=ERROR

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
new file mode 100644
index 0000000..9e8a5d2
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Ozonefs test
+Library             OperatingSystem
+Suite Setup         Startup Ozone cluster with size          5
+Suite Teardown      Teardown Ozone cluster
+Resource            ../commonlib.robot
+
+*** Variables ***
+${COMPOSEFILE}          ${CURDIR}/docker-compose.yaml
+${PROJECTDIR}           ${CURDIR}/../../../../../..
+
+
+*** Test Cases ***
+Create volume and bucket
+    Execute on          datanode        ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
+    Execute on          datanode        ozone oz -createBucket http://ksm/fstest/bucket1
+
+Check volume from ozonefs
+    ${result} =         Execute on          hadooplast        hdfs dfs -ls o3://bucket1.fstest/
+
+Create directory from ozonefs
+                        Execute on          hadooplast        hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
+    ${result} =         Execute on          ksm               ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+                                            Should contain    ${result}         testdir/deep

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
new file mode 100644
index 0000000..c3de4d1
--- /dev/null
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-ozone</artifactId>
+    <version>0.2.1-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-ozone-filesystem</artifactId>
+  <name>Apache Hadoop Ozone FileSystem</name>
+  <packaging>jar</packaging>
+  <version>0.2.1-SNAPSHOT</version>
+  <properties>
+    <file.encoding>UTF-8</file.encoding>
+    <downloadSources>true</downloadSources>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>3.1.1</version>
+        <configuration>
+          <artifactSet>
+            <includes>
+              <include>com.google.guava:guava:jar</include>
+              <include>org.slf4j:slf4j-api:jar</include>
+              <include>com.google.protobuf:protobuf-java</include>
+              <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+              <include>com.github.stephenc.jcip:jcip-annotations</include>
+              <include>com.google.code.findbugs:jsr305:jar</include>
+              <include>org.apache.hadoop:hadoop-ozone-client</include>
+              <include>org.apache.hadoop:hadoop-hdds-client</include>
+              <include>org.apache.hadoop:hadoop-hdds-common</include>
+              <include>org.fusesource.leveldbjni:leveldbjni-all</include>
+              <include>org.apache.ratis:ratis-server</include>
+              <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+              <include>com.google.auto.value:auto-value-annotations</include>
+              <include>com.squareup:javapoet:jar</include>
+              <include>org.jctools:jctools-core</include>
+              <include>org.apache.ratis:ratis-common</include>
+              <include>org.apache.ratis:ratis-client</include>
+              <include>org.apache.ratis:ratis-netty</include>
+              <include>org.apache.ratis:ratis-grpc</include>
+              <include>org.rocksdb:rocksdbjni</include>
+              <include>org.apache.hadoop:hadoop-ozone-common</include>
+            </includes>
+          </artifactSet>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <!-- build a shellprofile -->
+              <outputFile>
+                ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+              </outputFile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-server-scm</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-server-framework</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-ozone-manager</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-container-service</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdds-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-objectstore-service</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-integration-test</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-distcp</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-distcp</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
new file mode 100644
index 0000000..832a0cb
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+/**
+ * Constants for Ozone FileSystem implementation.
+ */
+public final class Constants {
+
+  public static final String OZONE_DEFAULT_USER = "hdfs";
+
+  public static final String OZONE_USER_DIR = "/user";
+
+  /** Local buffer directory. */
+  public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
+
+  /** Temporary directory. */
+  public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
+
+  /** Page size for Ozone listing operation. */
+  public static final int LISTING_PAGE_SIZE = 1024;
+
+  private Constants() {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
new file mode 100644
index 0000000..4163c13
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * ozone implementation of AbstractFileSystem.
+ * This impl delegates to the OzoneFileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class OzFs extends DelegateToFileSystem {
+
+  public OzFs(URI theUri, Configuration conf)
+      throws IOException, URISyntaxException {
+    super(theUri, new OzoneFileSystem(), conf,
+        OzoneConsts.OZONE_URI_SCHEME, false);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
new file mode 100644
index 0000000..4c5c0c8
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * The input stream for Ozone file system.
+ *
+ * TODO: Make inputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneFSInputStream extends FSInputStream {
+
+  private final ChunkGroupInputStream inputStream;
+
+  public OzoneFSInputStream(InputStream inputStream) {
+    this.inputStream = (ChunkGroupInputStream)inputStream;
+  }
+
+  @Override
+  public int read() throws IOException {
+    return inputStream.read();
+  }
+
+  @Override
+  public int read(byte[] b, int off, int len) throws IOException {
+    return inputStream.read(b, off, len);
+  }
+
+  @Override
+  public synchronized void close() throws IOException {
+    inputStream.close();
+  }
+
+  @Override
+  public void seek(long pos) throws IOException {
+    inputStream.seek(pos);
+  }
+
+  @Override
+  public long getPos() throws IOException {
+    return inputStream.getPos();
+  }
+
+  @Override
+  public boolean seekToNewSource(long targetPos) throws IOException {
+    return false;
+  }
+
+  @Override
+  public int available() throws IOException {
+    return inputStream.available();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
new file mode 100644
index 0000000..faa3628
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
+
+
+/**
+ * The output stream for Ozone file system.
+ *
+ * TODO: Make outputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+public class OzoneFSOutputStream extends OutputStream {
+
+  private final ChunkGroupOutputStream outputStream;
+
+  public OzoneFSOutputStream(OutputStream outputStream) {
+    this.outputStream = (ChunkGroupOutputStream)outputStream;
+  }
+
+  @Override
+  public void write(int b) throws IOException {
+    outputStream.write(b);
+  }
+
+  @Override
+  public void write(byte[] b, int off, int len) throws IOException {
+    outputStream.write(b, off, len);
+  }
+
+  @Override
+  public synchronized void flush() throws IOException {
+    outputStream.flush();
+  }
+
+  @Override
+  public synchronized void close() throws IOException {
+    outputStream.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
new file mode 100644
index 0000000..6906a9d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+
+/**
+ * The Ozone Filesystem implementation.
+ *
+ * This subclass is marked as private as code should not be creating it
+ * directly; use {@link FileSystem#get(Configuration)} and variants to create
+ * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
+ * accessed. Consider those private and unstable.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class OzoneFileSystem extends FileSystem {
+  static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
+
+  /** The Ozone client for connecting to Ozone server. */
+  private OzoneClient ozoneClient;
+  private ObjectStore objectStore;
+  private OzoneVolume volume;
+  private OzoneBucket bucket;
+  private URI uri;
+  private String userName;
+  private Path workingDir;
+  private ReplicationType replicationType;
+  private ReplicationFactor replicationFactor;
+
+  private static final Pattern URL_SCHEMA_PATTERN =
+      Pattern.compile("(.+)\\.([^\\.]+)");
+
+  @Override
+  public void initialize(URI name, Configuration conf) throws IOException {
+    super.initialize(name, conf);
+    setConf(conf);
+    Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+    assert getScheme().equals(name.getScheme());
+
+    String authority = name.getAuthority();
+
+    Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
+
+    if (!matcher.matches()) {
+      throw new IllegalArgumentException("Ozone file system url should be "
+          + "in the form o3://bucket.volume");
+    }
+    String bucketStr = matcher.group(1);
+    String volumeStr = matcher.group(2);
+
+    try {
+      uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
+          .setHost(authority).build();
+      LOG.trace("Ozone URI for ozfs initialization is " + uri);
+      this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
+      objectStore = ozoneClient.getObjectStore();
+      this.volume = objectStore.getVolume(volumeStr);
+      this.bucket = volume.getBucket(bucketStr);
+      this.replicationType = ReplicationType.valueOf(
+          conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+              OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
+      this.replicationFactor = ReplicationFactor.valueOf(
+          conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+              OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
+      try {
+        this.userName =
+            UserGroupInformation.getCurrentUser().getShortUserName();
+      } catch (IOException e) {
+        this.userName = OZONE_DEFAULT_USER;
+      }
+      this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+              .makeQualified(this.uri, this.workingDir);
+    } catch (URISyntaxException ue) {
+      final String msg = "Invalid Ozone endpoint " + name;
+      LOG.error(msg, ue);
+      throw new IOException(msg, ue);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      ozoneClient.close();
+    } finally {
+      super.close();
+    }
+  }
+
+  @Override
+  public URI getUri() {
+    return uri;
+  }
+
+  @Override
+  public String getScheme() {
+    return OZONE_URI_SCHEME;
+  }
+
+  @Override
+  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+    LOG.trace("open() path:{}", f);
+    final FileStatus fileStatus = getFileStatus(f);
+    final String key = pathToKey(f);
+    if (fileStatus.isDirectory()) {
+      throw new FileNotFoundException("Can't open directory " + f + " to read");
+    }
+
+    return new FSDataInputStream(
+        new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
+  }
+
+  @Override
+  public FSDataOutputStream create(Path f, FsPermission permission,
+                                   boolean overwrite, int bufferSize,
+                                   short replication, long blockSize,
+                                   Progressable progress) throws IOException {
+    LOG.trace("create() path:{}", f);
+    final String key = pathToKey(f);
+    final FileStatus status;
+    try {
+      status = getFileStatus(f);
+      if (status.isDirectory()) {
+        throw new FileAlreadyExistsException(f + " is a directory");
+      } else {
+        if (!overwrite) {
+          // path references a file and overwrite is disabled
+          throw new FileAlreadyExistsException(f + " already exists");
+        }
+        LOG.trace("Overwriting file {}", f);
+        deleteObject(key);
+      }
+    } catch (FileNotFoundException ignored) {
+      // check if the parent directory needs to be created
+      Path parent = f.getParent();
+      try {
+        // create all the directories for the parent
+        FileStatus parentStatus = getFileStatus(parent);
+        LOG.trace("parent key:{} status:{}", key, parentStatus);
+      } catch (FileNotFoundException e) {
+        mkdirs(parent);
+      }
+      // This exception needs to ignored as this means that the file currently
+      // does not exists and a new file can thus be created.
+    }
+
+    OzoneOutputStream ozoneOutputStream =
+        bucket.createKey(key, 0, replicationType, replicationFactor);
+    // We pass null to FSDataOutputStream so it won't count writes that
+    // are being buffered to a file
+    return new FSDataOutputStream(
+        new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
+  }
+
+  @Override
+  public FSDataOutputStream createNonRecursive(Path path,
+      FsPermission permission,
+      EnumSet<CreateFlag> flags,
+      int bufferSize,
+      short replication,
+      long blockSize,
+      Progressable progress) throws IOException {
+    final Path parent = path.getParent();
+    if (parent != null) {
+      // expect this to raise an exception if there is no parent
+      if (!getFileStatus(parent).isDirectory()) {
+        throw new FileAlreadyExistsException("Not a directory: " + parent);
+      }
+    }
+    return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
+        bufferSize, replication, blockSize, progress);
+  }
+
+  @Override
+  public FSDataOutputStream append(Path f, int bufferSize,
+      Progressable progress) throws IOException {
+    throw new UnsupportedOperationException("append() Not implemented by the "
+        + getClass().getSimpleName() + " FileSystem implementation");
+  }
+
+  private class RenameIterator extends OzoneListingIterator {
+    private final String srcKey;
+    private final String dstKey;
+
+    RenameIterator(Path srcPath, Path dstPath)
+        throws IOException {
+      super(srcPath);
+      srcKey = pathToKey(srcPath);
+      dstKey = pathToKey(dstPath);
+      LOG.trace("rename from:{} to:{}", srcKey, dstKey);
+    }
+
+    boolean processKey(String key) throws IOException {
+      String newKeyName = dstKey.concat(key.substring(srcKey.length()));
+      bucket.renameKey(key, newKeyName);
+      return true;
+    }
+  }
+
+  /**
+   * Check whether the source and destination path are valid and then perform
+   * rename from source path to destination path.
+   *
+   * The rename operation is performed by renaming the keys with src as prefix.
+   * For such keys the prefix is changed from src to dst.
+   *
+   * @param src source path for rename
+   * @param dst destination path for rename
+   * @return true if rename operation succeeded or
+   * if the src and dst have the same path and are of the same type
+   * @throws IOException on I/O errors or if the src/dst paths are invalid.
+   */
+  @Override
+  public boolean rename(Path src, Path dst) throws IOException {
+    if (src.equals(dst)) {
+      return true;
+    }
+
+    LOG.trace("rename() from:{} to:{}", src, dst);
+    if (src.isRoot()) {
+      // Cannot rename root of file system
+      LOG.trace("Cannot rename the root of a filesystem");
+      return false;
+    }
+
+    // Cannot rename a directory to its own subdirectory
+    Path dstParent = dst.getParent();
+    while (dstParent != null && !src.equals(dstParent)) {
+      dstParent = dstParent.getParent();
+    }
+    Preconditions.checkArgument(dstParent == null,
+        "Cannot rename a directory to its own subdirectory");
+    // Check if the source exists
+    FileStatus srcStatus;
+    try {
+      srcStatus = getFileStatus(src);
+    } catch (FileNotFoundException fnfe) {
+      // source doesn't exist, return
+      return false;
+    }
+
+    // Check if the destination exists
+    FileStatus dstStatus;
+    try {
+      dstStatus = getFileStatus(dst);
+    } catch (FileNotFoundException fnde) {
+      dstStatus = null;
+    }
+
+    if (dstStatus == null) {
+      // If dst doesn't exist, check whether dst parent dir exists or not
+      // if the parent exists, the source can still be renamed to dst path
+      dstStatus = getFileStatus(dst.getParent());
+      if (!dstStatus.isDirectory()) {
+        throw new IOException(String.format(
+            "Failed to rename %s to %s, %s is a file", src, dst,
+            dst.getParent()));
+      }
+    } else {
+      // if dst exists and source and destination are same,
+      // check both the src and dst are of same type
+      if (srcStatus.getPath().equals(dstStatus.getPath())) {
+        return !srcStatus.isDirectory();
+      } else if (dstStatus.isDirectory()) {
+        // If dst is a directory, rename source as subpath of it.
+        // for example rename /source to /dst will lead to /dst/source
+        dst = new Path(dst, src.getName());
+        FileStatus[] statuses;
+        try {
+          statuses = listStatus(dst);
+        } catch (FileNotFoundException fnde) {
+          statuses = null;
+        }
+
+        if (statuses != null && statuses.length > 0) {
+          // If dst exists and not a directory not empty
+          throw new FileAlreadyExistsException(String.format(
+              "Failed to rename %s to %s, file already exists or not empty!",
+              src, dst));
+        }
+      } else {
+        // If dst is not a directory
+        throw new FileAlreadyExistsException(String.format(
+            "Failed to rename %s to %s, file already exists!", src, dst));
+      }
+    }
+
+    if (srcStatus.isDirectory()) {
+      if (dst.toString().startsWith(src.toString())) {
+        LOG.trace("Cannot rename a directory to a subdirectory of self");
+        return false;
+      }
+    }
+    RenameIterator iterator = new RenameIterator(src, dst);
+    return iterator.iterate();
+  }
+
+  private class DeleteIterator extends OzoneListingIterator {
+    private boolean recursive;
+    DeleteIterator(Path f, boolean recursive)
+        throws IOException {
+      super(f);
+      this.recursive = recursive;
+      if (getStatus().isDirectory()
+          && !this.recursive
+          && listStatus(f).length != 0) {
+        throw new PathIsNotEmptyDirectoryException(f.toString());
+      }
+    }
+
+    boolean processKey(String key) throws IOException {
+      if (key.equals("")) {
+        LOG.trace("Skipping deleting root directory");
+        return true;
+      } else {
+        LOG.trace("deleting key:" + key);
+        boolean succeed = deleteObject(key);
+        // if recursive delete is requested ignore the return value of
+        // deleteObject and issue deletes for other keys.
+        return recursive || succeed;
+      }
+    }
+  }
+
+  @Override
+  public boolean delete(Path f, boolean recursive) throws IOException {
+    LOG.trace("delete() path:{} recursive:{}", f, recursive);
+    try {
+      DeleteIterator iterator = new DeleteIterator(f, recursive);
+      return iterator.iterate();
+    } catch (FileNotFoundException e) {
+      LOG.debug("Couldn't delete {} - does not exist", f);
+      return false;
+    }
+  }
+
+  private class ListStatusIterator extends OzoneListingIterator {
+    private  List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
+    private Path f;
+
+    ListStatusIterator(Path f) throws IOException  {
+      super(f);
+      this.f = f;
+    }
+
+    boolean processKey(String key) throws IOException {
+      Path keyPath = new Path(OZONE_URI_DELIMITER + key);
+      if (key.equals(getPathKey())) {
+        if (pathIsDirectory()) {
+          return true;
+        } else {
+          statuses.add(getFileStatus(keyPath));
+          return true;
+        }
+      }
+      // left with only subkeys now
+      if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
+        // skip keys which are for subdirectories of the directory
+        statuses.add(getFileStatus(keyPath));
+      }
+      return true;
+    }
+
+    FileStatus[] getStatuses() {
+      return statuses.toArray(new FileStatus[statuses.size()]);
+    }
+  }
+
+  @Override
+  public FileStatus[] listStatus(Path f) throws IOException {
+    LOG.trace("listStatus() path:{}", f);
+    ListStatusIterator iterator = new ListStatusIterator(f);
+    iterator.iterate();
+    return iterator.getStatuses();
+  }
+
+  @Override
+  public void setWorkingDirectory(Path newDir) {
+    workingDir = newDir;
+  }
+
+  @Override
+  public Path getWorkingDirectory() {
+    return workingDir;
+  }
+
+  /**
+   * Check whether the path is valid and then create directories.
+   * Directory is represented using a key with no value.
+   * All the non-existent parent directories are also created.
+   *
+   * @param path directory path to be created
+   * @return true if directory exists or created successfully.
+   * @throws IOException
+   */
+  private boolean mkdir(Path path) throws IOException {
+    Path fPart = path;
+    Path prevfPart = null;
+    do {
+      LOG.trace("validating path:{}", fPart);
+      try {
+        FileStatus fileStatus = getFileStatus(fPart);
+        if (fileStatus.isDirectory()) {
+          // If path exists and a directory, exit
+          break;
+        } else {
+          // Found a file here, rollback and delete newly created directories
+          LOG.trace("Found a file with same name as directory, path:{}", fPart);
+          if (prevfPart != null) {
+            delete(prevfPart, true);
+          }
+          throw new FileAlreadyExistsException(String.format(
+              "Can't make directory for path '%s', it is a file.", fPart));
+        }
+      } catch (FileNotFoundException fnfe) {
+        LOG.trace("creating directory for fpart:{}", fPart);
+        String key = pathToKey(fPart);
+        String dirKey = addTrailingSlashIfNeeded(key);
+        if (!createDirectory(dirKey)) {
+          // Directory creation failed here,
+          // rollback and delete newly created directories
+          LOG.trace("Directory creation failed, path:{}", fPart);
+          if (prevfPart != null) {
+            delete(prevfPart, true);
+          }
+          return false;
+        }
+      }
+      prevfPart = fPart;
+      fPart = fPart.getParent();
+    } while (fPart != null);
+    return true;
+  }
+
+  @Override
+  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+    LOG.trace("mkdir() path:{} ", f);
+    String key = pathToKey(f);
+    if (StringUtils.isEmpty(key)) {
+      return false;
+    }
+    return mkdir(f);
+  }
+
+  @Override
+  public FileStatus getFileStatus(Path f) throws IOException {
+    LOG.trace("getFileStatus() path:{}", f);
+    Path qualifiedPath = f.makeQualified(uri, workingDir);
+    String key = pathToKey(qualifiedPath);
+
+    if (key.length() == 0) {
+      return new FileStatus(0, true, 1, 0,
+          bucket.getCreationTime(), qualifiedPath);
+    }
+
+    // consider this a file and get key status
+    OzoneKey meta = getKeyInfo(key);
+    if (meta == null) {
+      key = addTrailingSlashIfNeeded(key);
+      meta = getKeyInfo(key);
+    }
+
+    if (meta == null) {
+      LOG.trace("File:{} not found", f);
+      throw new FileNotFoundException(f + ": No such file or directory!");
+    } else if (isDirectory(meta)) {
+      return new FileStatus(0, true, 1, 0,
+          meta.getModificationTime(), qualifiedPath);
+    } else {
+      //TODO: Fetch replication count from ratis config
+      return new FileStatus(meta.getDataSize(), false, 1,
+            getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
+    }
+  }
+
+  /**
+   * Helper method to fetch the key metadata info.
+   * @param key key whose metadata information needs to be fetched
+   * @return metadata info of the key
+   */
+  private OzoneKey getKeyInfo(String key) {
+    try {
+      return bucket.getKey(key);
+    } catch (IOException e) {
+      LOG.trace("Key:{} does not exists", key);
+      return null;
+    }
+  }
+
+  /**
+   * Helper method to check if an Ozone key is representing a directory.
+   * @param key key to be checked as a directory
+   * @return true if key is a directory, false otherwise
+   */
+  private boolean isDirectory(OzoneKey key) {
+    LOG.trace("key name:{} size:{}", key.getName(),
+        key.getDataSize());
+    return key.getName().endsWith(OZONE_URI_DELIMITER)
+        && (key.getDataSize() == 0);
+  }
+
+  /**
+   * Helper method to create an directory specified by key name in bucket.
+   * @param keyName key name to be created as directory
+   * @return true if the key is created, false otherwise
+   */
+  private boolean createDirectory(String keyName) {
+    try {
+      LOG.trace("creating dir for key:{}", keyName);
+      bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
+      return true;
+    } catch (IOException ioe) {
+      LOG.error("create key failed for key:{}", keyName, ioe);
+      return false;
+    }
+  }
+
+  /**
+   * Helper method to delete an object specified by key name in bucket.
+   * @param keyName key name to be deleted
+   * @return true if the key is deleted, false otherwise
+   */
+  private boolean deleteObject(String keyName) {
+    LOG.trace("issuing delete for key" + keyName);
+    try {
+      bucket.deleteKey(keyName);
+      return true;
+    } catch (IOException ioe) {
+      LOG.error("delete key failed " + ioe.getMessage());
+      return false;
+    }
+  }
+
+  /**
+   * Turn a path (relative or otherwise) into an Ozone key.
+   *
+   * @param path the path of the file.
+   * @return the key of the object that represents the file.
+   */
+  public String pathToKey(Path path) {
+    Objects.requireNonNull(path, "Path can not be null!");
+    if (!path.isAbsolute()) {
+      path = new Path(workingDir, path);
+    }
+    // removing leading '/' char
+    String key = path.toUri().getPath().substring(1);
+    LOG.trace("path for key:{} is:{}", key, path);
+    return key;
+  }
+
+  /**
+   * Add trailing delimiter to path if it is already not present.
+   *
+   * @param key the ozone Key which needs to be appended
+   * @return delimiter appended key
+   */
+  private String addTrailingSlashIfNeeded(String key) {
+    if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+      return key + OZONE_URI_DELIMITER;
+    } else {
+      return key;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "OzoneFileSystem{URI=" + uri + ", "
+        + "workingDir=" + workingDir + ", "
+        + "userName=" + userName + ", "
+        + "statistics=" + statistics
+        + "}";
+  }
+
+  private abstract class OzoneListingIterator {
+    private final Path path;
+    private final FileStatus status;
+    private String pathKey;
+    private Iterator<OzoneKey> keyIterator;
+
+    OzoneListingIterator(Path path)
+        throws IOException {
+      this.path = path;
+      this.status = getFileStatus(path);
+      this.pathKey = pathToKey(path);
+      if (status.isDirectory()) {
+        this.pathKey = addTrailingSlashIfNeeded(pathKey);
+      }
+      keyIterator = bucket.listKeys(pathKey);
+    }
+
+    abstract boolean processKey(String key) throws IOException;
+
+    // iterates all the keys in the particular path
+    boolean iterate() throws IOException {
+      LOG.trace("Iterating path {}", path);
+      if (status.isDirectory()) {
+        LOG.trace("Iterating directory:{}", pathKey);
+        while (keyIterator.hasNext()) {
+          OzoneKey key = keyIterator.next();
+          LOG.trace("iterating key:{}", key.getName());
+          if (!processKey(key.getName())) {
+            return false;
+          }
+        }
+        return true;
+      } else {
+        LOG.trace("iterating file:{}", path);
+        return processKey(pathKey);
+      }
+    }
+
+    String getPathKey() {
+      return pathKey;
+    }
+
+    boolean pathIsDirectory() {
+      return status.isDirectory();
+    }
+
+    FileStatus getStatus() {
+      return status;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
new file mode 100644
index 0000000..93e82c3
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Ozone Filesystem.
+ *
+ * Except for the exceptions, it should all be hidden as implementation details.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
new file mode 100644
index 0000000..ad21f28
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Test OzoneFSInputStream by reading through multiple interfaces.
+ */
+public class TestOzoneFSInputStream {
+  private static MiniOzoneCluster cluster = null;
+  private static FileSystem fs;
+  private static StorageHandler storageHandler;
+  private static Path filePath = null;
+  private static byte[] data = null;
+
+  /**
+   * Create a MiniDFSCluster for testing.
+   * <p>
+   * Ozone is made active by setting OZONE_ENABLED = true and
+   * OZONE_HANDLER_TYPE_KEY = "distributed"
+   *
+   * @throws IOException
+   */
+  @BeforeClass
+  public static void init() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+    cluster = MiniOzoneCluster.newBuilder(conf)
+        .setNumDatanodes(10)
+        .build();
+    cluster.waitForClusterToBeReady();
+    storageHandler =
+        new ObjectStoreHandler(conf).getStorageHandler();
+
+    // create a volume and a bucket to be used by OzoneFileSystem
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+        null, null, null, null);
+    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+    volumeArgs.setUserName(userName);
+    volumeArgs.setAdminName(adminName);
+    storageHandler.createVolume(volumeArgs);
+    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+    storageHandler.createBucket(bucketArgs);
+
+    // Fetch the host and port for File System init
+    DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
+        .getDatanodeDetails();
+    int port = datanodeDetails
+        .getPort(DatanodeDetails.Port.Name.REST).getValue();
+    String host = datanodeDetails.getHostName();
+
+    // Set the fs.defaultFS and start the filesystem
+    String uri = String.format("%s://%s.%s/",
+        OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
+    fs =  FileSystem.get(conf);
+    int fileLen = 100 * 1024 * 1024;
+    data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
+    filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
+    try (FSDataOutputStream stream = fs.create(filePath)) {
+      stream.write(data);
+    }
+  }
+
+  /**
+   * Shutdown MiniDFSCluster.
+   */
+  @AfterClass
+  public static void shutdown() throws IOException {
+    fs.close();
+    storageHandler.close();
+    cluster.shutdown();
+  }
+
+  @Test
+  public void testO3FSSingleByteRead() throws IOException {
+    FSDataInputStream inputStream = fs.open(filePath);
+    byte[] value = new byte[data.length];
+    int i = 0;
+    while(true) {
+      int val = inputStream.read();
+      if (val == -1) {
+        break;
+      }
+      value[i] = (byte)val;
+      Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
+      i++;
+    }
+    Assert.assertEquals(i, data.length);
+    Assert.assertTrue(Arrays.equals(value, data));
+    inputStream.close();
+  }
+
+  @Test
+  public void testO3FSMultiByteRead() throws IOException {
+    FSDataInputStream inputStream = fs.open(filePath);
+    byte[] value = new byte[data.length];
+    byte[] tmp = new byte[1* 1024 *1024];
+    int i = 0;
+    while(true) {
+      int val = inputStream.read(tmp);
+      if (val == -1) {
+        break;
+      }
+      System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
+      i++;
+    }
+    Assert.assertEquals(i * tmp.length, data.length);
+    Assert.assertTrue(Arrays.equals(value, data));
+    inputStream.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
new file mode 100644
index 0000000..a225702
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test OzoneFileSystem Interfaces.
+ *
+ * This test will test the various interfaces i.e.
+ * create, read, write, getFileStatus
+ */
+@RunWith(Parameterized.class)
+public class TestOzoneFileInterfaces {
+
+  private String rootPath;
+  private String userName;
+
+  /**
+   * Parameter class to set absolute url/defaultFS handling.
+   * <p>
+   * Hadoop file systems could be used in multiple ways: Using the defaultfs
+   * and file path without the schema, or use absolute url-s even with
+   * different defaultFS. This parameter matrix would test both the use cases.
+   */
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {{false, true}, {true, false}});
+  }
+
+  private boolean setDefaultFs;
+
+  private boolean useAbsolutePath;
+
+  private static MiniOzoneCluster cluster = null;
+
+  private static FileSystem fs;
+
+  private static StorageHandler storageHandler;
+
+  public TestOzoneFileInterfaces(boolean setDefaultFs,
+      boolean useAbsolutePath) {
+    this.setDefaultFs = setDefaultFs;
+    this.useAbsolutePath = useAbsolutePath;
+  }
+
+  @Before
+  public void init() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    cluster = MiniOzoneCluster.newBuilder(conf)
+        .setNumDatanodes(3)
+        .build();
+    cluster.waitForClusterToBeReady();
+    storageHandler =
+        new ObjectStoreHandler(conf).getStorageHandler();
+
+    // create a volume and a bucket to be used by OzoneFileSystem
+    userName = "user" + RandomStringUtils.randomNumeric(5);
+    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+        null, null, null, null);
+    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+    volumeArgs.setUserName(userName);
+    volumeArgs.setAdminName(adminName);
+    storageHandler.createVolume(volumeArgs);
+    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+    storageHandler.createBucket(bucketArgs);
+
+    rootPath = String
+        .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
+            volumeName);
+    if (setDefaultFs) {
+      // Set the fs.defaultFS and start the filesystem
+      conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+      fs = FileSystem.get(conf);
+    } else {
+      fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
+    }
+  }
+
+  @After
+  public void teardown() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+    IOUtils.closeQuietly(fs);
+    IOUtils.closeQuietly(storageHandler);
+  }
+
+  @Test
+  public void testFileSystemInit() throws IOException {
+    if (setDefaultFs) {
+      assertTrue(
+          "The initialized file system is not OzoneFileSystem but " +
+              fs.getClass(),
+          fs instanceof OzoneFileSystem);
+      assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
+    }
+  }
+
+  @Test
+  public void testOzFsReadWrite() throws IOException {
+    long currentTime = Time.now();
+    int stringLen = 20;
+    String data = RandomStringUtils.randomAlphanumeric(stringLen);
+    String filePath = RandomStringUtils.randomAlphanumeric(5);
+    Path path = createPath("/" + filePath);
+    try (FSDataOutputStream stream = fs.create(path)) {
+      stream.writeBytes(data);
+    }
+
+    FileStatus status = fs.getFileStatus(path);
+    // The timestamp of the newly created file should always be greater than
+    // the time when the test was started
+    assertTrue("Modification time has not been recorded: " + status,
+        status.getModificationTime() > currentTime);
+
+    try (FSDataInputStream inputStream = fs.open(path)) {
+      byte[] buffer = new byte[stringLen];
+      inputStream.readFully(0, buffer);
+      String out = new String(buffer, 0, buffer.length);
+      assertEquals(data, out);
+    }
+  }
+
+
+  @Test
+  public void testDirectory() throws IOException {
+    String dirPath = RandomStringUtils.randomAlphanumeric(5);
+    Path path = createPath("/" + dirPath);
+    assertTrue("Makedirs returned with false for the path " + path,
+        fs.mkdirs(path));
+
+    FileStatus status = fs.getFileStatus(path);
+    assertTrue("The created path is not directory.", status.isDirectory());
+
+    assertEquals(0, status.getLen());
+
+    FileStatus[] statusList = fs.listStatus(createPath("/"));
+    assertEquals(1, statusList.length);
+    assertEquals(status, statusList[0]);
+
+    FileStatus statusRoot = fs.getFileStatus(createPath("/"));
+    assertTrue("Root dir (/) is not a directory.", status.isDirectory());
+    assertEquals(0, status.getLen());
+
+
+  }
+
+  @Test
+  public void testPathToKey() throws Exception {
+    OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
+
+    assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
+
+    assertEquals("user/" + getCurrentUser() + "/key1/key2",
+        ozoneFs.pathToKey(new Path("key1/key2")));
+
+    assertEquals("key1/key2",
+        ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
+  }
+
+  private String getCurrentUser() {
+    try {
+      return UserGroupInformation.getCurrentUser().getShortUserName();
+    } catch (IOException e) {
+      return OZONE_DEFAULT_USER;
+    }
+  }
+
+  private Path createPath(String relativePath) {
+    if (useAbsolutePath) {
+      return new Path(
+          rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
+    } else {
+      return new Path(relativePath);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
new file mode 100644
index 0000000..dd54315
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests creating files.
+ */
+public class ITestOzoneContractCreate extends AbstractContractCreateTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
new file mode 100644
index 0000000..f0a3d8d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering deletes.
+ */
+public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
new file mode 100644
index 0000000..134a9ad
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ * Uses the block output stream, buffered to disk. This is the
+ * recommended output mechanism for DistCP due to its scalability.
+ */
+public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected OzoneContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
new file mode 100644
index 0000000..98bbb14
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering getFileStatus.
+ */
+public class ITestOzoneContractGetFileStatus
+    extends AbstractContractGetFileStatusTest {
+
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+
+  @Override
+  public void teardown() throws Exception {
+    getLog().info("FS details {}", getFileSystem());
+    super.teardown();
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    return super.createConfiguration();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
new file mode 100644
index 0000000..bc0de5d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Test dir operations on Ozone.
+ */
+public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
new file mode 100644
index 0000000..0bc57d4
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests opening files.
+ */
+public class ITestOzoneContractOpen extends AbstractContractOpenTest {
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
new file mode 100644
index 0000000..8ce1d1b
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering rename.
+ */
+public class ITestOzoneContractRename extends AbstractContractRenameTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[43/50] [abbrv] hadoop git commit: HDFS-13707. [PROVIDED Storage] Fix failing integration tests in ITestProvidedImplementation. Contributed by Virajith Jalaparti.

Posted by gi...@apache.org.
HDFS-13707. [PROVIDED Storage] Fix failing integration tests in ITestProvidedImplementation. Contributed by Virajith Jalaparti.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73746c5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73746c5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73746c5d

Branch: refs/heads/HADOOP-15461
Commit: 73746c5da76d5e39df131534a1ec35dfc5d2529b
Parents: e4d7227
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 29 09:56:13 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 29 09:56:13 2018 -0700

----------------------------------------------------------------------
 .../common/blockaliasmap/impl/TextFileRegionAliasMap.java       | 5 +++--
 .../hdfs/server/namenode/ITestProvidedImplementation.java       | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/73746c5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
index abe92e3..4d65142 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
@@ -28,6 +28,7 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Collections;
@@ -359,7 +360,7 @@ public class TextFileRegionAliasMap
       }
       byte[] nonce = new byte[0];
       if (f.length == 6) {
-        nonce = f[5].getBytes(Charset.forName("UTF-8"));
+        nonce = Base64.getDecoder().decode(f[5]);
       }
       return new FileRegion(Long.parseLong(f[0]), new Path(f[1]),
           Long.parseLong(f[2]), Long.parseLong(f[3]), Long.parseLong(f[4]),
@@ -451,7 +452,7 @@ public class TextFileRegionAliasMap
       out.append(Long.toString(block.getGenerationStamp()));
       if (psl.getNonce().length > 0) {
         out.append(delim)
-            .append(new String(psl.getNonce(), Charset.forName("UTF-8")));
+            .append(Base64.getEncoder().encodeToString(psl.getNonce()));
       }
       out.append("\n");
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/73746c5d/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
index 49c9bcf..7d3ab0e 100644
--- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
+++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
@@ -132,7 +132,7 @@ public class ITestProvidedImplementation {
         nnDirPath.toString());
     conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_READ_FILE,
         new Path(nnDirPath, fileNameFromBlockPoolID(bpid)).toString());
-    conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_DELIMITER, ",");
+    conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_DELIMITER, "\t");
 
     conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR_PROVIDED,
         new File(providedPath.toUri()).toString());


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[29/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton.

Posted by gi...@apache.org.
HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56a4cdb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56a4cdb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56a4cdb9

Branch: refs/heads/HADOOP-15461
Commit: 56a4cdb9804daea7164155a5b1b4eba44a11b705
Parents: 0d6fe5f
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:28:00 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:28:00 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  26 --
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   1 -
 .../common/src/main/resources/ozone-default.xml |  47 ---
 .../hdds/scm/container/ContainerMapping.java    |  10 +-
 .../replication/ContainerSupervisor.java        | 340 -------------------
 .../container/replication/InProgressPool.java   | 255 --------------
 .../scm/container/replication/PeriodicPool.java | 119 -------
 .../scm/container/replication/package-info.java |  23 --
 .../hadoop/hdds/scm/node/NodeManager.java       |   6 -
 .../hadoop/hdds/scm/node/NodePoolManager.java   |  71 ----
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  23 --
 .../hdds/scm/node/SCMNodePoolManager.java       | 269 ---------------
 .../hdds/scm/container/MockNodeManager.java     |   6 -
 .../hdds/scm/node/TestSCMNodePoolManager.java   | 160 ---------
 .../testutils/ReplicationNodeManagerMock.java   |   5 -
 .../ReplicationNodePoolManagerMock.java         | 133 --------
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  31 --
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  74 ----
 18 files changed, 3 insertions(+), 1596 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 85407e6..df6fbf0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,32 +243,6 @@ public final class ScmConfigKeys {
   public static final String
       OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
 
-  /**
-   * Don't start processing a pool if we have not had a minimum number of
-   * seconds from the last processing.
-   */
-  public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
-      "ozone.scm.container.report.processing.interval";
-  public static final String
-      OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
-  /**
-   * This determines the total number of pools to be processed in parallel.
-   */
-  public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
-      "ozone.scm.max.nodepool.processing.threads";
-  public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
-  /**
-   * These 2 settings control the number of threads in executor pool and time
-   * outs for thw container reports from all nodes.
-   */
-  public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
-      "ozone.scm.max.container.report.threads";
-  public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
-      "ozone.scm.container.reports.wait.timeout";
-  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
-      "5m";
 
   public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
       "ozone.scm.block.deletion.max.retry";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index c40dc8e..08a5ffd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,7 +91,6 @@ public final class OzoneConsts {
   public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
   public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
   public static final String BLOCK_DB = "block.db";
-  public static final String NODEPOOL_DB = "nodepool.db";
   public static final String OPEN_CONTAINERS_DB = "openContainers.db";
   public static final String DELETED_BLOCK_DB = "deletedBlock.db";
   public static final String KSM_DB_NAME = "ksm.db";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7a91610..25365c8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,25 +572,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.container.report.processing.interval</name>
-    <value>60s</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>Time interval for scm to process container reports
-      for a node pool. Scm handles node pool reports in a cyclic clock
-      manner, it fetches pools periodically with this time interval.
-    </description>
-  </property>
-  <property>
-    <name>ozone.scm.container.reports.wait.timeout</name>
-    <value>300s</value>
-    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
-    <description>Maximum time to wait in seconds for processing all container
-      reports from
-      a node pool. It determines the timeout for a
-      node pool report.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.container.size.gb</name>
     <value>5</value>
     <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -793,17 +774,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.container.report.threads</name>
-    <value>100</value>
-    <tag>OZONE, PERFORMANCE</tag>
-    <description>
-      Maximum number of threads to process container reports in scm.
-      Each container report from a data node is processed by scm in a worker
-      thread, fetched from a thread pool. This property is used to control the
-      maximum size of the thread pool.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.max.hb.count.to.process</name>
     <value>5000</value>
     <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -815,14 +785,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
-    <description>
-      Number of node pools to process in parallel.
-    </description>
-  </property>
-  <property>
     <name>ozone.scm.names</name>
     <value/>
     <tag>OZONE</tag>
@@ -844,15 +806,6 @@
     </description>
   </property>
   <property>
-    <name>ozone.scm.max.nodepool.processing.threads</name>
-    <value>1</value>
-    <tag>OZONE, SCM</tag>
-    <description>
-      Controls the number of node pools that can be processed in parallel by
-      Container Supervisor.
-    </description>
-  </property>
-  <property>
     <name>ozone.trace.enabled</name>
     <value>false</value>
     <tag>OZONE, DEBUG</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index b563e90..9fd30f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -80,7 +79,6 @@ public class ContainerMapping implements Mapping {
   private final PipelineSelector pipelineSelector;
   private final ContainerStateManager containerStateManager;
   private final LeaseManager<ContainerInfo> containerLeaseManager;
-  private final ContainerSupervisor containerSupervisor;
   private final float containerCloseThreshold;
   private final ContainerCloser closer;
   private final long size;
@@ -127,9 +125,7 @@ public class ContainerMapping implements Mapping {
         OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
     this.containerStateManager =
         new ContainerStateManager(conf, this);
-    this.containerSupervisor =
-        new ContainerSupervisor(conf, nodeManager,
-            nodeManager.getNodePoolManager());
+
     this.containerCloseThreshold = conf.getFloat(
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -407,8 +403,8 @@ public class ContainerMapping implements Mapping {
       throws IOException {
     List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
         containerInfos = reports.getReportsList();
-    containerSupervisor.handleContainerReport(datanodeDetails, reports);
-    for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+
+     for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
         containerInfos) {
       byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
       lock.lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index 5bd0574..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
-    .sleepUninterruptibly;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(ContainerSupervisor.class);
-
-  private final NodePoolManager poolManager;
-  private final HashSet<String> poolNames;
-  private final PriorityQueue<PeriodicPool> poolQueue;
-  private final NodeManager nodeManager;
-  private final long containerProcessingLag;
-  private final AtomicBoolean runnable;
-  private final ExecutorService executorService;
-  private final long maxPoolWait;
-  private long poolProcessCount;
-  private final List<InProgressPool> inProgressPoolList;
-  private final AtomicInteger threadFaultCount;
-  private final int inProgressPoolMaxCount;
-
-  private final ReadWriteLock inProgressPoolListLock;
-
-  /**
-   * Returns the number of times we have processed pools.
-   * @return long
-   */
-  public long getPoolProcessCount() {
-    return poolProcessCount;
-  }
-
-
-  /**
-   * Constructs a class that computes Replication Levels.
-   *
-   * @param conf - OzoneConfiguration
-   * @param nodeManager - Node Manager
-   * @param poolManager - Pool Manager
-   */
-  public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
-                             NodePoolManager poolManager) {
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(nodeManager);
-    this.containerProcessingLag =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
-            OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
-            TimeUnit.SECONDS
-        ) * 1000;
-    int maxContainerReportThreads =
-        conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
-            OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
-        );
-    this.maxPoolWait =
-        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
-            OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
-            TimeUnit.MILLISECONDS);
-    this.inProgressPoolMaxCount = conf.getInt(
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
-        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
-    this.poolManager = poolManager;
-    this.nodeManager = nodeManager;
-    this.poolNames = new HashSet<>();
-    this.poolQueue = new PriorityQueue<>();
-    this.runnable = new AtomicBoolean(true);
-    this.threadFaultCount = new AtomicInteger(0);
-    this.executorService = newCachedThreadPool(
-        new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("Container Reports Processing Thread - %d")
-            .build(), maxContainerReportThreads);
-    this.inProgressPoolList = new LinkedList<>();
-    this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
-    initPoolProcessThread();
-  }
-
-  private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
-      int maxThreads) {
-    return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
-        new LinkedBlockingQueue<>(), threadFactory);
-  }
-
-  /**
-   * Returns the number of pools that are under process right now.
-   * @return  int - Number of pools that are in process.
-   */
-  public int getInProgressPoolCount() {
-    return inProgressPoolList.size();
-  }
-
-  /**
-   * Exits the background thread.
-   */
-  public void setExit() {
-    this.runnable.set(false);
-  }
-
-  /**
-   * Adds or removes pools from names that we need to process.
-   *
-   * There are two different cases that we need to process.
-   * The case where some pools are being added and some times we have to
-   * handle cases where pools are removed.
-   */
-  private void refreshPools() {
-    List<String> pools = this.poolManager.getNodePools();
-    if (pools != null) {
-
-      HashSet<String> removedPools =
-          computePoolDifference(this.poolNames, new HashSet<>(pools));
-
-      HashSet<String> addedPools =
-          computePoolDifference(new HashSet<>(pools), this.poolNames);
-      // TODO: Support remove pool API in pool manager so that this code
-      // path can be tested. This never happens in the current code base.
-      for (String poolName : removedPools) {
-        for (PeriodicPool periodicPool : poolQueue) {
-          if (periodicPool.getPoolName().compareTo(poolName) == 0) {
-            poolQueue.remove(periodicPool);
-          }
-        }
-      }
-      // Remove the pool names that we have in the list.
-      this.poolNames.removeAll(removedPools);
-
-      for (String poolName : addedPools) {
-        poolQueue.add(new PeriodicPool(poolName));
-      }
-
-      // Add to the pool names we are tracking.
-      poolNames.addAll(addedPools);
-    }
-
-  }
-
-  /**
-   * Handle the case where pools are added.
-   *
-   * @param newPools - New Pools list
-   * @param oldPool - oldPool List.
-   */
-  private HashSet<String> computePoolDifference(HashSet<String> newPools,
-      Set<String> oldPool) {
-    Preconditions.checkNotNull(newPools);
-    Preconditions.checkNotNull(oldPool);
-    HashSet<String> newSet = new HashSet<>(newPools);
-    newSet.removeAll(oldPool);
-    return newSet;
-  }
-
-  private void initPoolProcessThread() {
-
-    /*
-     * Task that runs to check if we need to start a pool processing job.
-     * if so we create a pool reconciliation job and find out of all the
-     * expected containers are on the nodes.
-     */
-    Runnable processPools = () -> {
-      while (runnable.get()) {
-        // Make sure that we don't have any new pools.
-        refreshPools();
-        while (inProgressPoolList.size() < inProgressPoolMaxCount) {
-          PeriodicPool pool = poolQueue.poll();
-          if (pool != null) {
-            if (pool.getLastProcessedTime() + this.containerProcessingLag >
-                Time.monotonicNow()) {
-              LOG.debug("Not within the time window for processing: {}",
-                  pool.getPoolName());
-              // we might over sleep here, not a big deal.
-              sleepUninterruptibly(this.containerProcessingLag,
-                  TimeUnit.MILLISECONDS);
-            }
-            LOG.debug("Adding pool {} to container processing queue",
-                pool.getPoolName());
-            InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
-                pool, this.nodeManager, this.poolManager, this.executorService);
-            inProgressPool.startReconciliation();
-            inProgressPoolListLock.writeLock().lock();
-            try {
-              inProgressPoolList.add(inProgressPool);
-            } finally {
-              inProgressPoolListLock.writeLock().unlock();
-            }
-            poolProcessCount++;
-          } else {
-            break;
-          }
-        }
-        sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
-        inProgressPoolListLock.readLock().lock();
-        try {
-          for (InProgressPool inProgressPool : inProgressPoolList) {
-            inProgressPool.finalizeReconciliation();
-            poolQueue.add(inProgressPool.getPool());
-          }
-        } finally {
-          inProgressPoolListLock.readLock().unlock();
-        }
-        inProgressPoolListLock.writeLock().lock();
-        try {
-          inProgressPoolList.clear();
-        } finally {
-          inProgressPoolListLock.writeLock().unlock();
-        }
-      }
-    };
-
-    // We will have only one thread for pool processing.
-    Thread poolProcessThread = new Thread(processPools);
-    poolProcessThread.setDaemon(true);
-    poolProcessThread.setName("Pool replica thread");
-    poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
-      // Let us just restart this thread after logging a critical error.
-      // if this thread is not running we cannot handle commands from SCM.
-      LOG.error("Critical Error : Pool replica thread encountered an " +
-          "error. Thread: {} Error Count : {}", t.toString(), e,
-          threadFaultCount.incrementAndGet());
-      poolProcessThread.start();
-      // TODO : Add a config to restrict how many times we will restart this
-      // thread in a single session.
-    });
-    poolProcessThread.start();
-  }
-
-  /**
-   * Adds a container report to appropriate inProgress Pool.
-   * @param containerReport  -- Container report for a specific container from
-   * a datanode.
-   */
-  public void handleContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto containerReport) {
-    inProgressPoolListLock.readLock().lock();
-    try {
-      String poolName = poolManager.getNodePool(datanodeDetails);
-      for (InProgressPool ppool : inProgressPoolList) {
-        if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
-          ppool.handleContainerReport(datanodeDetails, containerReport);
-          return;
-        }
-      }
-      // TODO: Decide if we can do anything else with this report.
-      LOG.debug("Discarding the container report for pool {}. " +
-              "That pool is not currently in the pool reconciliation process." +
-              " Container Name: {}", poolName, datanodeDetails);
-    } catch (SCMException e) {
-      LOG.warn("Skipping processing container report from datanode {}, "
-              + "cause: failed to get the corresponding node pool",
-          datanodeDetails.toString(), e);
-    } finally {
-      inProgressPoolListLock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get in process pool list, used for testing.
-   * @return List of InProgressPool
-   */
-  @VisibleForTesting
-  public List<InProgressPool> getInProcessPoolList() {
-    return inProgressPoolList;
-  }
-
-  /**
-   * Shutdown the Container Replication Manager.
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    setExit();
-    HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 4b54731..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(InProgressPool.class);
-
-  private final PeriodicPool pool;
-  private final NodeManager nodeManager;
-  private final NodePoolManager poolManager;
-  private final ExecutorService executorService;
-  private final Map<Long, Integer> containerCountMap;
-  private final Map<UUID, Boolean> processedNodeSet;
-  private final long startTime;
-  private ProgressStatus status;
-  private AtomicInteger nodeCount;
-  private AtomicInteger nodeProcessed;
-  private AtomicInteger containerProcessedCount;
-  private long maxWaitTime;
-  /**
-   * Constructs an pool that is being processed.
-   *  @param maxWaitTime - Maximum wait time in milliseconds.
-   * @param pool - Pool that we are working against
-   * @param nodeManager - Nodemanager
-   * @param poolManager - pool manager
-   * @param executorService - Shared Executor service.
-   */
-  InProgressPool(long maxWaitTime, PeriodicPool pool,
-      NodeManager nodeManager, NodePoolManager poolManager,
-                 ExecutorService executorService) {
-    Preconditions.checkNotNull(pool);
-    Preconditions.checkNotNull(nodeManager);
-    Preconditions.checkNotNull(poolManager);
-    Preconditions.checkNotNull(executorService);
-    Preconditions.checkArgument(maxWaitTime > 0);
-    this.pool = pool;
-    this.nodeManager = nodeManager;
-    this.poolManager = poolManager;
-    this.executorService = executorService;
-    this.containerCountMap = new ConcurrentHashMap<>();
-    this.processedNodeSet = new ConcurrentHashMap<>();
-    this.maxWaitTime = maxWaitTime;
-    startTime = Time.monotonicNow();
-  }
-
-  /**
-   * Returns periodic pool.
-   *
-   * @return PeriodicPool
-   */
-  public PeriodicPool getPool() {
-    return pool;
-  }
-
-  /**
-   * We are done if we have got reports from all nodes or we have
-   * done waiting for the specified time.
-   *
-   * @return true if we are done, false otherwise.
-   */
-  public boolean isDone() {
-    return (nodeCount.get() == nodeProcessed.get()) ||
-        (this.startTime + this.maxWaitTime) > Time.monotonicNow();
-  }
-
-  /**
-   * Gets the number of containers processed.
-   *
-   * @return int
-   */
-  public int getContainerProcessedCount() {
-    return containerProcessedCount.get();
-  }
-
-  /**
-   * Returns the start time in milliseconds.
-   *
-   * @return - Start Time.
-   */
-  public long getStartTime() {
-    return startTime;
-  }
-
-  /**
-   * Get the number of nodes in this pool.
-   *
-   * @return - node count
-   */
-  public int getNodeCount() {
-    return nodeCount.get();
-  }
-
-  /**
-   * Get the number of nodes that we have already processed container reports
-   * from.
-   *
-   * @return - Processed count.
-   */
-  public int getNodeProcessed() {
-    return nodeProcessed.get();
-  }
-
-  /**
-   * Returns the current status.
-   *
-   * @return Status
-   */
-  public ProgressStatus getStatus() {
-    return status;
-  }
-
-  /**
-   * Starts the reconciliation process for all the nodes in the pool.
-   */
-  public void startReconciliation() {
-    List<DatanodeDetails> datanodeDetailsList =
-        this.poolManager.getNodes(pool.getPoolName());
-    if (datanodeDetailsList.size() == 0) {
-      LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
-          pool.getPoolName());
-      this.status = ProgressStatus.Error;
-      return;
-    }
-
-    nodeProcessed = new AtomicInteger(0);
-    containerProcessedCount = new AtomicInteger(0);
-    nodeCount = new AtomicInteger(0);
-    this.status = ProgressStatus.InProgress;
-    this.getPool().setLastProcessedTime(Time.monotonicNow());
-  }
-
-  /**
-   * Queues a container Report for handling. This is done in a worker thread
-   * since decoding a container report might be compute intensive . We don't
-   * want to block since we have asked for bunch of container reports
-   * from a set of datanodes.
-   *
-   * @param containerReport - ContainerReport
-   */
-  public void handleContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto containerReport) {
-    if (status == ProgressStatus.InProgress) {
-      executorService.submit(processContainerReport(datanodeDetails,
-          containerReport));
-    } else {
-      LOG.debug("Cannot handle container report when the pool is in {} status.",
-          status);
-    }
-  }
-
-  private Runnable processContainerReport(DatanodeDetails datanodeDetails,
-      ContainerReportsProto reports) {
-    return () -> {
-      if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
-          (k) -> true)) {
-        nodeProcessed.incrementAndGet();
-        LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
-            datanodeDetails.getUuid());
-        for (ContainerInfo info : reports.getReportsList()) {
-          containerProcessedCount.incrementAndGet();
-          LOG.debug("Total Containers processed: {} Container Name: {}",
-              containerProcessedCount.get(), info.getContainerID());
-
-          // Update the container map with count + 1 if the key exists or
-          // update the map with 1. Since this is a concurrentMap the
-          // computation and update is atomic.
-          containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
-        }
-      }
-    };
-  }
-
-  /**
-   * Filter the containers based on specific rules.
-   *
-   * @param predicate -- Predicate to filter by
-   * @return A list of map entries.
-   */
-  public List<Map.Entry<Long, Integer>> filterContainer(
-      Predicate<Map.Entry<Long, Integer>> predicate) {
-    return containerCountMap.entrySet().stream()
-        .filter(predicate).collect(Collectors.toList());
-  }
-
-  /**
-   * Used only for testing, calling this will abort container report
-   * processing. This is very dangerous call and should not be made by any users
-   */
-  @VisibleForTesting
-  public void setDoneProcessing() {
-    nodeProcessed.set(nodeCount.get());
-  }
-
-  /**
-   * Returns the pool name.
-   *
-   * @return Name of the pool.
-   */
-  String getPoolName() {
-    return pool.getPoolName();
-  }
-
-  public void finalizeReconciliation() {
-    status = ProgressStatus.Done;
-    //TODO: Add finalizing logic. This is where actual reconciliation happens.
-  }
-
-  /**
-   * Current status of the computing replication status.
-   */
-  public enum ProgressStatus {
-    InProgress, Done, Error
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index ef28aa7..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
-  private final String poolName;
-  private long lastProcessedTime;
-  private AtomicLong totalProcessedCount;
-
-  /**
-   * Constructs a periodic pool.
-   *
-   * @param poolName - Name of the pool
-   */
-  public PeriodicPool(String poolName) {
-    this.poolName = poolName;
-    lastProcessedTime = 0;
-    totalProcessedCount = new AtomicLong(0);
-  }
-
-  /**
-   * Get pool Name.
-   * @return PoolName
-   */
-  public String getPoolName() {
-    return poolName;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(PeriodicPool o) {
-    return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    PeriodicPool that = (PeriodicPool) o;
-
-    return poolName.equals(that.poolName);
-  }
-
-  @Override
-  public int hashCode() {
-    return poolName.hashCode();
-  }
-
-  /**
-   * Returns the Total Times we have processed this pool.
-   *
-   * @return processed count.
-   */
-  public long getTotalProcessedCount() {
-    return totalProcessedCount.get();
-  }
-
-  /**
-   * Gets the last time we processed this pool.
-   * @return time in milliseconds
-   */
-  public long getLastProcessedTime() {
-    return this.lastProcessedTime;
-  }
-
-
-  /**
-   * Sets the last processed time.
-   *
-   * @param lastProcessedTime - Long in milliseconds.
-   */
-
-  public void setLastProcessedTime(long lastProcessedTime) {
-    this.lastProcessedTime = lastProcessedTime;
-  }
-
-  /*
-   * Increments the total processed count.
-   */
-  public void incTotalProcessedCount() {
-    this.totalProcessedCount.incrementAndGet();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
deleted file mode 100644
index 7bbe2ef..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 4392633..72d7e94 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,12 +124,6 @@ public interface NodeManager extends StorageContainerNodeProtocol,
   SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
 
   /**
-   * Returns the NodePoolManager associated with the NodeManager.
-   * @return NodePoolManager
-   */
-  NodePoolManager getNodePoolManager();
-
-  /**
    * Wait for the heartbeat is processed by NodeManager.
    * @return true if heartbeat has been processed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
deleted file mode 100644
index 46faf9ca..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
-  /**
-   * Add a node to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   */
-  void addNode(String pool, DatanodeDetails node) throws IOException;
-
-  /**
-   * Remove a node from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   * @throws SCMException
-   */
-  void removeNode(String pool, DatanodeDetails node)
-      throws SCMException;
-
-  /**
-   * Get a list of known node pools.
-   * @return a list of known node pool names or an empty list if not node pool
-   * is defined.
-   */
-  List<String> getNodePools();
-
-  /**
-   * Get all nodes of a node pool given the name of the node pool.
-   * @param pool - name of the node pool.
-   * @return a list of datanode ids or an empty list if the node pool was not
-   *  found.
-   */
-  List<DatanodeDetails> getNodes(String pool);
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeDetails - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   */
-  String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fc8b013..adca8ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
-import com.google.protobuf.GeneratedMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -159,7 +158,6 @@ public class SCMNodeManager
   private ObjectName nmInfoBean;
 
   // Node pool manager.
-  private final SCMNodePoolManager nodePoolManager;
   private final StorageContainerManager scmManager;
 
   public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -210,7 +208,6 @@ public class SCMNodeManager
 
     registerMXBean();
 
-    this.nodePoolManager = new SCMNodePoolManager(conf);
     this.scmManager = scmManager;
   }
 
@@ -682,7 +679,6 @@ public class SCMNodeManager
   @Override
   public void close() throws IOException {
     unregisterMXBean();
-    nodePoolManager.close();
     executorService.shutdown();
     try {
       if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -760,20 +756,6 @@ public class SCMNodeManager
       LOG.info("Leaving startup chill mode.");
     }
 
-    // TODO: define node pool policy for non-default node pool.
-    // For now, all nodes are added to the "DefaultNodePool" upon registration
-    // if it has not been added to any node pool yet.
-    try {
-      if (nodePoolManager.getNodePool(datanodeDetails) == null) {
-        nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
-            datanodeDetails);
-      }
-    } catch (IOException e) {
-      // TODO: make sure registration failure is handled correctly.
-      return RegisteredCommand.newBuilder()
-          .setErrorCode(ErrorCode.errorNodeNotPermitted)
-          .build();
-    }
     // Updating Node Report, as registration is successful
     updateNodeStat(datanodeDetails.getUuid(), nodeReport);
     LOG.info("Data node with ID: {} Registered.",
@@ -860,11 +842,6 @@ public class SCMNodeManager
   }
 
   @Override
-  public NodePoolManager getNodePoolManager() {
-    return nodePoolManager;
-  }
-
-  @Override
   public Map<String, Integer> getNodeCount() {
     Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
     for(NodeState state : NodeState.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index faf330e..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
-    .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(SCMNodePoolManager.class);
-  private static final List<DatanodeDetails> EMPTY_NODE_LIST =
-      new ArrayList<>();
-  private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
-  public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
-  // DB that saves the node to node pool mapping.
-  private MetadataStore nodePoolStore;
-
-  // In-memory node pool to nodes mapping
-  private HashMap<String, Set<DatanodeDetails>> nodePools;
-
-  // Read-write lock for nodepool operations
-  private ReadWriteLock lock;
-
-  /**
-   * Construct SCMNodePoolManager class that manages node to node pool mapping.
-   * @param conf - configuration.
-   * @throws IOException
-   */
-  public SCMNodePoolManager(final OzoneConfiguration conf)
-      throws IOException {
-    final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
-        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-    File metaDir = getOzoneMetaDirPath(conf);
-    String scmMetaDataDir = metaDir.getPath();
-    File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
-    nodePoolStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf)
-        .setDbFile(nodePoolDBPath)
-        .setCacheSize(cacheSize * OzoneConsts.MB)
-        .build();
-    nodePools = new HashMap<>();
-    lock = new ReentrantReadWriteLock();
-    init();
-  }
-
-  /**
-   * Initialize the in-memory store based on persist store from level db.
-   * No lock is needed as init() is only invoked by constructor.
-   * @throws SCMException
-   */
-  private void init() throws SCMException {
-    try {
-      nodePoolStore.iterate(null, (key, value) -> {
-        try {
-          DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
-              HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
-          String poolName = DFSUtil.bytes2String(value);
-
-          Set<DatanodeDetails> nodePool = null;
-          if (nodePools.containsKey(poolName)) {
-            nodePool = nodePools.get(poolName);
-          } else {
-            nodePool = new HashSet<>();
-            nodePools.put(poolName, nodePool);
-          }
-          nodePool.add(nodeId);
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Adding node: {} to node pool: {}",
-                nodeId, poolName);
-          }
-        } catch (IOException e) {
-          LOG.warn("Can't add a datanode to node pool, continue next...");
-        }
-        return true;
-      });
-    } catch (IOException e) {
-      LOG.error("Loading node pool error " + e);
-      throw new SCMException("Failed to load node pool",
-          FAILED_TO_LOAD_NODEPOOL);
-    }
-  }
-
-  /**
-   * Add a datanode to a node pool.
-   * @param pool - name of the node pool.
-   * @param node - name of the datanode.
-   */
-  @Override
-  public void addNode(final String pool, final DatanodeDetails node)
-      throws IOException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // add to the persistent store
-      nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
-          DFSUtil.string2Bytes(pool));
-
-      // add to the in-memory store
-      Set<DatanodeDetails> nodePool = null;
-      if (nodePools.containsKey(pool)) {
-        nodePool = nodePools.get(pool);
-      } else {
-        nodePool = new HashSet<DatanodeDetails>();
-        nodePools.put(pool, nodePool);
-      }
-      nodePool.add(node);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Remove a datanode from a node pool.
-   * @param pool - name of the node pool.
-   * @param node - datanode id.
-   * @throws SCMException
-   */
-  @Override
-  public void removeNode(final String pool, final DatanodeDetails node)
-      throws SCMException {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    Preconditions.checkNotNull(node, "node is null");
-    lock.writeLock().lock();
-    try {
-      // Remove from the persistent store
-      byte[] kName = node.getProtoBufMessage().toByteArray();
-      byte[] kData = nodePoolStore.get(kName);
-      if (kData == null) {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);
-      }
-      nodePoolStore.delete(kName);
-
-      // Remove from the in-memory store
-      if (nodePools.containsKey(pool)) {
-        Set<DatanodeDetails> nodePool = nodePools.get(pool);
-        nodePool.remove(node);
-      } else {
-        throw new SCMException(String.format("Unable to find node %s from" +
-            " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
-            FAILED_TO_FIND_NODE_IN_POOL);
-      }
-    } catch (IOException e) {
-      throw new SCMException("Failed to remove node " + node.toString()
-          + " from node pool " + pool, e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    } finally {
-      lock.writeLock().unlock();
-    }
-  }
-
-  /**
-   * Get all the node pools.
-   * @return all the node pools.
-   */
-  @Override
-  public List<String> getNodePools() {
-    lock.readLock().lock();
-    try {
-      if (!nodePools.isEmpty()) {
-        return nodePools.keySet().stream().collect(Collectors.toList());
-      } else {
-        return EMPTY_NODEPOOL_LIST;
-      }
-    } finally {
-      lock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get all datanodes of a specific node pool.
-   * @param pool - name of the node pool.
-   * @return all datanodes of the specified node pool.
-   */
-  @Override
-  public List<DatanodeDetails> getNodes(final String pool) {
-    Preconditions.checkNotNull(pool, "pool name is null");
-    if (nodePools.containsKey(pool)) {
-      return nodePools.get(pool).stream().collect(Collectors.toList());
-    } else {
-      return EMPTY_NODE_LIST;
-    }
-  }
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   * @param datanodeDetails - datanode ID.
-   * @return node pool name if it has been assigned.
-   * null if the node has not been assigned to any node pool yet.
-   * TODO: Put this in a in-memory map if performance is an issue.
-   */
-  @Override
-  public String getNodePool(final DatanodeDetails datanodeDetails)
-      throws SCMException {
-    Preconditions.checkNotNull(datanodeDetails, "node is null");
-    try {
-      byte[] result = nodePoolStore.get(
-          datanodeDetails.getProtoBufMessage().toByteArray());
-      return result == null ? null : DFSUtil.bytes2String(result);
-    } catch (IOException e) {
-      throw new SCMException("Failed to get node pool for node "
-          + datanodeDetails.toString(), e,
-          SCMException.ResultCodes.IO_EXCEPTION);
-    }
-  }
-
-  /**
-   * Close node pool level db store.
-   * @throws IOException
-   */
-  @Override
-  public void close() throws IOException {
-    nodePoolStore.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 8c59462..80b5d6e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.container;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -273,11 +272,6 @@ public class MockNodeManager implements NodeManager {
     return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
   }
 
-  @Override
-  public NodePoolManager getNodePoolManager() {
-    return Mockito.mock(NodePoolManager.class);
-  }
-
   /**
    * Used for testing.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
deleted file mode 100644
index 8f412de..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for SCM node pool manager.
- */
-public class TestSCMNodePoolManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestSCMNodePoolManager.class);
-
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  private final File testDir = PathUtils.getTestDir(
-      TestSCMNodePoolManager.class);
-
-  SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
-      throws IOException {
-    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
-        testDir.getAbsolutePath());
-    conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
-        SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
-    return new SCMNodePoolManager(conf);
-  }
-
-  /**
-   * Test default node pool.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testDefaultNodePool() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    try {
-      final String defaultPool = "DefaultPool";
-      NodePoolManager npMgr = createNodePoolManager(conf);
-
-      final int nodeCount = 4;
-      final List<DatanodeDetails> nodes = TestUtils
-          .getListOfDatanodeDetails(nodeCount);
-      assertEquals(0, npMgr.getNodePools().size());
-      for (DatanodeDetails node: nodes) {
-        npMgr.addNode(defaultPool, node);
-      }
-      List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-      assertEquals(nodeCount, nodesRetrieved.size());
-      assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-
-      DatanodeDetails nodeRemoved = nodes.remove(2);
-      npMgr.removeNode(defaultPool, nodeRemoved);
-      List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
-      assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
-
-      List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
-      assertEquals(0, nonExistSet.size());
-    } finally {
-      FileUtil.fullyDelete(testDir);
-    }
-  }
-
-
-  /**
-   * Test default node pool reload.
-   *
-   * @throws IOException
-   */
-  @Test
-  public void testDefaultNodePoolReload() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    final String defaultPool = "DefaultPool";
-    final int nodeCount = 4;
-    final List<DatanodeDetails> nodes = TestUtils
-        .getListOfDatanodeDetails(nodeCount);
-
-    try {
-      try {
-        SCMNodePoolManager npMgr = createNodePoolManager(conf);
-        assertEquals(0, npMgr.getNodePools().size());
-        for (DatanodeDetails node : nodes) {
-          npMgr.addNode(defaultPool, node);
-        }
-        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-        assertEquals(nodeCount, nodesRetrieved.size());
-        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-        npMgr.close();
-      } finally {
-        LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
-            " and close.");
-      }
-
-      // try reload with a new NodePoolManager instance
-      try {
-        SCMNodePoolManager npMgr = createNodePoolManager(conf);
-        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
-        assertEquals(nodeCount, nodesRetrieved.size());
-        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-      } finally {
-        LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
-      }
-    } finally {
-      FileUtil.fullyDelete(testDir);
-    }
-  }
-
-  /**
-   * Compare and verify that two datanode lists are equal.
-   * @param list1 - datanode list 1.
-   * @param list2 - datanode list 2.
-   */
-  private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
-      List<DatanodeDetails> list2) {
-    assertEquals(list1.size(), list2.size());
-    Collections.sort(list1);
-    Collections.sort(list2);
-    assertTrue(ListUtils.isEqualList(list1, list2));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 072d821..1a4dcd7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.CommandQueue;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -201,10 +200,6 @@ public class ReplicationNodeManagerMock implements NodeManager {
     return null;
   }
 
-  @Override
-  public NodePoolManager getNodePoolManager() {
-    return Mockito.mock(NodePoolManager.class);
-  }
 
   /**
    * Wait for the heartbeat is processed by NodeManager.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
deleted file mode 100644
index ffcd752..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.testutils;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Pool Manager replication mock.
- */
-public class ReplicationNodePoolManagerMock implements NodePoolManager {
-
-  private final Map<DatanodeDetails, String> nodeMemberShip;
-
-  /**
-   * A node pool manager for testing.
-   */
-  public ReplicationNodePoolManagerMock() {
-    nodeMemberShip = new HashMap<>();
-  }
-
-  /**
-   * Add a node to a node pool.
-   *
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   */
-  @Override
-  public void addNode(String pool, DatanodeDetails node) {
-    nodeMemberShip.put(node, pool);
-  }
-
-  /**
-   * Remove a node from a node pool.
-   *
-   * @param pool - name of the node pool.
-   * @param node - data node.
-   * @throws SCMException
-   */
-  @Override
-  public void removeNode(String pool, DatanodeDetails node)
-      throws SCMException {
-    nodeMemberShip.remove(node);
-
-  }
-
-  /**
-   * Get a list of known node pools.
-   *
-   * @return a list of known node pool names or an empty list if not node pool
-   * is defined.
-   */
-  @Override
-  public List<String> getNodePools() {
-    Set<String> poolSet = new HashSet<>();
-    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
-      poolSet.add(entry.getValue());
-    }
-    return new ArrayList<>(poolSet);
-
-  }
-
-  /**
-   * Get all nodes of a node pool given the name of the node pool.
-   *
-   * @param pool - name of the node pool.
-   * @return a list of datanode ids or an empty list if the node pool was not
-   * found.
-   */
-  @Override
-  public List<DatanodeDetails> getNodes(String pool) {
-    Set<DatanodeDetails> datanodeSet = new HashSet<>();
-    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
-      if (entry.getValue().equals(pool)) {
-        datanodeSet.add(entry.getKey());
-      }
-    }
-    return new ArrayList<>(datanodeSet);
-  }
-
-  /**
-   * Get the node pool name if the node has been added to a node pool.
-   *
-   * @param datanodeDetails DatanodeDetails.
-   * @return node pool name if it has been assigned. null if the node has not
-   * been assigned to any node pool yet.
-   */
-  @Override
-  public String getNodePool(DatanodeDetails datanodeDetails) {
-    return nodeMemberShip.get(datanodeDetails);
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated
-   * with it. If the stream is already closed then invoking this
-   * method has no effect.
-   * <p>
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the
-   * close may fail require careful attention. It is strongly advised
-   * to relinquish the underlying resources and to internally
-   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
-   * the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 4d70af8..b4ed2b1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,12 +51,9 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.UUID;
 
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * This class tests the CLI that transforms container into SQLite DB files.
@@ -177,34 +174,6 @@ public class TestContainerSQLCli {
   }
 
   @Test
-  public void testConvertNodepoolDB() throws Exception {
-    String dbOutPath = GenericTestUtils.getTempPath(
-        UUID.randomUUID() + "/out_sql.db");
-    String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
-    String dbPath = dbRootPath + "/" + NODEPOOL_DB;
-    String[] args = {"-p", dbPath, "-o", dbOutPath};
-
-    cli.run(args);
-
-    // verify the sqlite db
-    HashMap<String, String> expectedPool = new HashMap<>();
-    for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
-      expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
-    }
-    Connection conn = connectDB(dbOutPath);
-    String sql = "SELECT * FROM nodePool";
-    ResultSet rs = executeQuery(conn, sql);
-    while(rs.next()) {
-      String datanodeUUID = rs.getString("datanodeUUID");
-      String poolName = rs.getString("poolName");
-      assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
-    }
-    assertEquals(0, expectedPool.size());
-
-    Files.delete(Paths.get(dbOutPath));
-  }
-
-  @Test
   public void testConvertContainerDB() throws Exception {
     String dbOutPath = GenericTestUtils.getTempPath(
         UUID.randomUUID() + "/out_sql.db");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index 2bd43fb..edc0d7b 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.scm.cli;
 
 import com.google.common.base.Preconditions;
 import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
 import org.apache.commons.cli.BasicParser;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
@@ -60,13 +59,11 @@ import java.sql.Statement;
 import java.util.HashSet;
 import java.util.Set;
 
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
 
 /**
@@ -111,15 +108,6 @@ public class SQLCLI  extends Configured implements Tool {
   private static final String INSERT_CONTAINER_MEMBERS =
       "INSERT INTO containerMembers (containerName, datanodeUUID) " +
           "VALUES (\"%s\", \"%s\")";
-  // for nodepool.db
-  private static final String CREATE_NODE_POOL =
-      "CREATE TABLE nodePool (" +
-          "datanodeUUID TEXT NOT NULL," +
-          "poolName TEXT NOT NULL," +
-          "PRIMARY KEY(datanodeUUID, poolName))";
-  private static final String INSERT_NODE_POOL =
-      "INSERT INTO nodePool (datanodeUUID, poolName) " +
-          "VALUES (\"%s\", \"%s\")";
   // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
   // for openContainer.db
   private static final String CREATE_OPEN_CONTAINER =
@@ -285,9 +273,6 @@ public class SQLCLI  extends Configured implements Tool {
     if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
       LOG.info("Converting container DB");
       convertContainerDB(dbPath, outPath);
-    }  else if (dbName.toString().equals(NODEPOOL_DB)) {
-      LOG.info("Converting node pool DB");
-      convertNodePoolDB(dbPath, outPath);
     } else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
       LOG.info("Converting open container DB");
       convertOpenContainerDB(dbPath, outPath);
@@ -543,66 +528,7 @@ public class SQLCLI  extends Configured implements Tool {
     }
     LOG.info("Insertion completed.");
   }
-  /**
-   * Converts nodePool.db to sqlite. The schema of sql db:
-   * two tables, nodePool and datanodeInfo (the same datanode Info as for
-   * container.db).
-   *
-   * nodePool
-   * ---------------------------------------------------------
-   * datanodeUUID* | poolName*
-   * ---------------------------------------------------------
-   *
-   * datanodeInfo:
-   * ---------------------------------------------------------
-   * hostname | datanodeUUid* | xferPort | ipcPort
-   * ---------------------------------------------------------
-   *
-   * --------------------------------
-   * |containerPort
-   * --------------------------------
-   *
-   * @param dbPath path to container db.
-   * @param outPath path to output sqlite
-   * @throws IOException throws exception.
-   */
-  private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
-    LOG.info("Create table for sql node pool db.");
-    File dbFile = dbPath.toFile();
-    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
-        .setConf(conf).setDbFile(dbFile).build();
-        Connection conn = connectDB(outPath.toString())) {
-      executeSQL(conn, CREATE_NODE_POOL);
-      executeSQL(conn, CREATE_DATANODE_INFO);
 
-      dbStore.iterate(null, (key, value) -> {
-        DatanodeDetails nodeId = DatanodeDetails
-            .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
-                .PARSER.parseFrom(key));
-        String blockPool = DFSUtil.bytes2String(value);
-        try {
-          insertNodePoolDB(conn, blockPool, nodeId);
-          return true;
-        } catch (SQLException e) {
-          throw new IOException(e);
-        }
-      });
-    }
-  }
-
-  private void insertNodePoolDB(Connection conn, String blockPool,
-      DatanodeDetails datanodeDetails) throws SQLException {
-    String insertNodePool = String.format(INSERT_NODE_POOL,
-        datanodeDetails.getUuidString(), blockPool);
-    executeSQL(conn, insertNodePool);
-
-    String insertDatanodeDetails = String
-        .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
-            datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
-            datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
-                .getValue());
-    executeSQL(conn, insertDatanodeDetails);
-  }
 
   /**
    * Convert openContainer.db to sqlite db file. This is rather simple db,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[07/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable ozonefs artifact Contributed by Marton, Elek.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
deleted file mode 100644
index 3156eb2..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-import java.io.IOException;
-
-/**
- * Ozone contract test for ROOT directory operations.
- */
-public class ITestOzoneContractRootDir extends
-    AbstractContractRootDirectoryTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
deleted file mode 100644
index c4bc0ff..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering file seek.
- */
-public class ITestOzoneContractSeek extends AbstractContractSeekTest {
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
deleted file mode 100644
index 8417e46..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Assert;
-
-import java.io.IOException;
-
-/**
- * The contract of Ozone: only enabled if the test bucket is provided.
- */
-class OzoneContract extends AbstractFSContract {
-
-  private static MiniOzoneCluster cluster;
-  private static StorageHandler storageHandler;
-  private static final String CONTRACT_XML = "contract/ozone.xml";
-
-  OzoneContract(Configuration conf) {
-    super(conf);
-    //insert the base features
-    addConfResource(CONTRACT_XML);
-  }
-
-  @Override
-  public String getScheme() {
-    return OzoneConsts.OZONE_URI_SCHEME;
-  }
-
-  @Override
-  public Path getTestPath() {
-    Path path = new Path("/test");
-    return path;
-  }
-
-  public static void createCluster() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    conf.addResource(CONTRACT_XML);
-
-    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
-    try {
-      cluster.waitForClusterToBeReady();
-    } catch (Exception e) {
-      throw new IOException(e);
-    }
-    storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
-  }
-
-  private void copyClusterConfigs(String configKey) {
-    getConf().set(configKey, cluster.getConf().get(configKey));
-  }
-
-  @Override
-  public FileSystem getTestFileSystem() throws IOException {
-    //assumes cluster is not null
-    Assert.assertNotNull("cluster not created", cluster);
-
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
-
-    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
-        null, null, null, null);
-    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
-    volumeArgs.setUserName(userName);
-    volumeArgs.setAdminName(adminName);
-    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
-    try {
-      storageHandler.createVolume(volumeArgs);
-      storageHandler.createBucket(bucketArgs);
-    } catch (OzoneException e) {
-      throw new IOException(e.getMessage());
-    }
-
-    String uri = String.format("%s://%s.%s/",
-        OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
-    getConf().set("fs.defaultFS", uri);
-    copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
-    copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-    return FileSystem.get(getConf());
-  }
-
-  public static void destroyCluster() throws IOException {
-    if (cluster != null) {
-      cluster.shutdown();
-      cluster = null;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml b/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
deleted file mode 100644
index fe2075c..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<!--
-  ~ Licensed to the Apache Software Foundation (ASF) under one
-  ~  or more contributor license agreements.  See the NOTICE file
-  ~  distributed with this work for additional information
-  ~  regarding copyright ownership.  The ASF licenses this file
-  ~  to you under the Apache License, Version 2.0 (the
-  ~  "License"); you may not use this file except in compliance
-  ~  with the License.  You may obtain a copy of the License at
-  ~
-  ~       http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~  Unless required by applicable law or agreed to in writing, software
-  ~  distributed under the License is distributed on an "AS IS" BASIS,
-  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~  See the License for the specific language governing permissions and
-  ~  limitations under the License.
-  -->
-
-<configuration>
-  <!--
-  Ozone is a blobstore, with very different behavior than a classic filesystem.
-  -->
-
-    <property>
-        <name>fs.contract.test.root-tests-enabled</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.test.random-seek-count</name>
-        <value>10</value>
-    </property>
-
-    <property>
-        <name>fs.contract.is-blobstore</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.create-visibility-delayed</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.is-case-sensitive</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.rename-returns-false-if-source-missing</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.rename-remove-dest-if-empty-dir</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-append</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-atomic-directory-delete</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-atomic-rename</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-block-locality</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-concat</name>
-        <value>false</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-getfilestatus</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-seek</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-seek-on-closed-file</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.rejects-seek-past-eof</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-strict-exceptions</name>
-        <value>true</value>
-    </property>
-
-    <property>
-        <name>fs.contract.supports-unix-permissions</name>
-        <value>false</value>
-    </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
deleted file mode 100644
index 3bf1619..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-
-# for debugging low level Ozone operations, uncomment this line
-# log4j.logger.org.apache.hadoop.ozone=DEBUG

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 21cc7ce..42ce94c 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -192,20 +192,5 @@
         </plugins>
       </build>
     </profile>
-
-    <profile>
-      <id>hdds</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-ozone-filesystem</artifactId>
-          <scope>compile</scope>
-          <version>${project.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
   </profiles>
 </project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index f421e58..dca59d3 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -67,15 +67,4 @@
       </plugin>
     </plugins>
   </build>
-  <profiles>
-    <profile>
-      <id>hdds</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <modules>
-        <module>hadoop-ozone</module>
-      </modules>
-    </profile>
-  </profiles>
 </project>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[27/50] [abbrv] hadoop git commit: Revert "HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index edc0d7b..2bd43fb 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.scm.cli;
 
 import com.google.common.base.Preconditions;
 import com.google.common.primitives.Longs;
+import com.google.protobuf.ByteString;
 import org.apache.commons.cli.BasicParser;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
@@ -59,11 +60,13 @@ import java.sql.Statement;
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
 import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
 
 /**
@@ -108,6 +111,15 @@ public class SQLCLI  extends Configured implements Tool {
   private static final String INSERT_CONTAINER_MEMBERS =
       "INSERT INTO containerMembers (containerName, datanodeUUID) " +
           "VALUES (\"%s\", \"%s\")";
+  // for nodepool.db
+  private static final String CREATE_NODE_POOL =
+      "CREATE TABLE nodePool (" +
+          "datanodeUUID TEXT NOT NULL," +
+          "poolName TEXT NOT NULL," +
+          "PRIMARY KEY(datanodeUUID, poolName))";
+  private static final String INSERT_NODE_POOL =
+      "INSERT INTO nodePool (datanodeUUID, poolName) " +
+          "VALUES (\"%s\", \"%s\")";
   // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
   // for openContainer.db
   private static final String CREATE_OPEN_CONTAINER =
@@ -273,6 +285,9 @@ public class SQLCLI  extends Configured implements Tool {
     if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
       LOG.info("Converting container DB");
       convertContainerDB(dbPath, outPath);
+    }  else if (dbName.toString().equals(NODEPOOL_DB)) {
+      LOG.info("Converting node pool DB");
+      convertNodePoolDB(dbPath, outPath);
     } else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
       LOG.info("Converting open container DB");
       convertOpenContainerDB(dbPath, outPath);
@@ -528,7 +543,66 @@ public class SQLCLI  extends Configured implements Tool {
     }
     LOG.info("Insertion completed.");
   }
+  /**
+   * Converts nodePool.db to sqlite. The schema of sql db:
+   * two tables, nodePool and datanodeInfo (the same datanode Info as for
+   * container.db).
+   *
+   * nodePool
+   * ---------------------------------------------------------
+   * datanodeUUID* | poolName*
+   * ---------------------------------------------------------
+   *
+   * datanodeInfo:
+   * ---------------------------------------------------------
+   * hostname | datanodeUUid* | xferPort | ipcPort
+   * ---------------------------------------------------------
+   *
+   * --------------------------------
+   * |containerPort
+   * --------------------------------
+   *
+   * @param dbPath path to container db.
+   * @param outPath path to output sqlite
+   * @throws IOException throws exception.
+   */
+  private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
+    LOG.info("Create table for sql node pool db.");
+    File dbFile = dbPath.toFile();
+    try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+        .setConf(conf).setDbFile(dbFile).build();
+        Connection conn = connectDB(outPath.toString())) {
+      executeSQL(conn, CREATE_NODE_POOL);
+      executeSQL(conn, CREATE_DATANODE_INFO);
 
+      dbStore.iterate(null, (key, value) -> {
+        DatanodeDetails nodeId = DatanodeDetails
+            .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
+                .PARSER.parseFrom(key));
+        String blockPool = DFSUtil.bytes2String(value);
+        try {
+          insertNodePoolDB(conn, blockPool, nodeId);
+          return true;
+        } catch (SQLException e) {
+          throw new IOException(e);
+        }
+      });
+    }
+  }
+
+  private void insertNodePoolDB(Connection conn, String blockPool,
+      DatanodeDetails datanodeDetails) throws SQLException {
+    String insertNodePool = String.format(INSERT_NODE_POOL,
+        datanodeDetails.getUuidString(), blockPool);
+    executeSQL(conn, insertNodePool);
+
+    String insertDatanodeDetails = String
+        .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
+            datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
+            datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
+                .getValue());
+    executeSQL(conn, insertDatanodeDetails);
+  }
 
   /**
    * Convert openContainer.db to sqlite db file. This is rather simple db,


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[33/50] [abbrv] hadoop git commit: HDDS-193. Make Datanode heartbeat dispatcher in SCM event based. Contributed by Elek, Marton.

Posted by gi...@apache.org.
HDDS-193. Make Datanode heartbeat dispatcher in SCM event based.
Contributed by Elek, Marton.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8752a485
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8752a485
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8752a485

Branch: refs/heads/HADOOP-15461
Commit: 8752a48564028cb5892c19e29d4e5b984d70c076
Parents: 1893271
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Jun 27 14:18:25 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 14:18:25 2018 -0700

----------------------------------------------------------------------
 .../server/SCMDatanodeHeartbeatDispatcher.java  | 126 +++++++++
 .../scm/server/SCMDatanodeProtocolServer.java   |  14 +-
 .../scm/server/StorageContainerManager.java     |   4 +-
 .../SCMDatanodeContainerReportHandler.java      |  76 ------
 .../report/SCMDatanodeHeartbeatDispatcher.java  | 189 --------------
 .../report/SCMDatanodeNodeReportHandler.java    |  43 ----
 .../server/report/SCMDatanodeReportHandler.java |  83 ------
 .../report/SCMDatanodeReportHandlerFactory.java |  82 ------
 .../hdds/scm/server/report/package-info.java    |  57 -----
 .../TestSCMDatanodeHeartbeatDispatcher.java     | 119 +++++++++
 .../TestSCMDatanodeContainerReportHandler.java  |  34 ---
 .../TestSCMDatanodeHeartbeatDispatcher.java     | 138 ----------
 .../TestSCMDatanodeNodeReportHandler.java       |  36 ---
 .../TestSCMDatanodeReportHandlerFactory.java    |  51 ----
 .../hdds/scm/server/report/package-info.java    |  21 --
 .../apache/hadoop/ozone/scm/TestSCMMetrics.java | 253 -------------------
 16 files changed, 254 insertions(+), 1072 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
new file mode 100644
index 0000000..36f10a9
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
+
+import com.google.protobuf.GeneratedMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is responsible for dispatching heartbeat from datanode to
+ * appropriate EventHandler at SCM.
+ */
+public final class SCMDatanodeHeartbeatDispatcher {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SCMDatanodeHeartbeatDispatcher.class);
+
+  private EventPublisher eventPublisher;
+
+  public static final TypedEvent<NodeReportFromDatanode> NODE_REPORT =
+      new TypedEvent<>(NodeReportFromDatanode.class);
+
+  public static final TypedEvent<ContainerReportFromDatanode> CONTAINER_REPORT =
+      new TypedEvent<ContainerReportFromDatanode>(ContainerReportFromDatanode.class);
+
+  public SCMDatanodeHeartbeatDispatcher(EventPublisher eventPublisher) {
+    this.eventPublisher = eventPublisher;
+  }
+
+
+  /**
+   * Dispatches heartbeat to registered event handlers.
+   *
+   * @param heartbeat heartbeat to be dispatched.
+   */
+  public void dispatch(SCMHeartbeatRequestProto heartbeat) {
+    DatanodeDetails datanodeDetails =
+        DatanodeDetails.getFromProtoBuf(heartbeat.getDatanodeDetails());
+
+    if (heartbeat.hasNodeReport()) {
+      eventPublisher.fireEvent(NODE_REPORT,
+          new NodeReportFromDatanode(datanodeDetails,
+              heartbeat.getNodeReport()));
+    }
+
+    if (heartbeat.hasContainerReport()) {
+      eventPublisher.fireEvent(CONTAINER_REPORT,
+          new ContainerReportFromDatanode(datanodeDetails,
+              heartbeat.getContainerReport()));
+
+    }
+  }
+
+  /**
+   * Wrapper class for events with the datanode origin.
+   */
+  public static class ReportFromDatanode<T extends GeneratedMessage> {
+
+    private final DatanodeDetails datanodeDetails;
+
+    private final T report;
+
+    public ReportFromDatanode(DatanodeDetails datanodeDetails, T report) {
+      this.datanodeDetails = datanodeDetails;
+      this.report = report;
+    }
+
+    public DatanodeDetails getDatanodeDetails() {
+      return datanodeDetails;
+    }
+
+    public T getReport() {
+      return report;
+    }
+  }
+
+  /**
+   * Node report event payload with origin.
+   */
+  public static class NodeReportFromDatanode
+      extends ReportFromDatanode<NodeReportProto> {
+
+    public NodeReportFromDatanode(DatanodeDetails datanodeDetails,
+        NodeReportProto report) {
+      super(datanodeDetails, report);
+    }
+  }
+
+  /**
+   * Container report event payload with origin.
+   */
+  public static class ContainerReportFromDatanode
+      extends ReportFromDatanode<ContainerReportsProto> {
+
+    public ContainerReportFromDatanode(DatanodeDetails datanodeDetails,
+        ContainerReportsProto report) {
+      super(datanodeDetails, report);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index eb5ce1a..56b0719 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -73,7 +73,7 @@ import static org.apache.hadoop.hdds.protocol.proto
 
 
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.server.report.SCMDatanodeHeartbeatDispatcher;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.RPC;
@@ -122,14 +122,19 @@ public class SCMDatanodeProtocolServer implements
   private final SCMDatanodeHeartbeatDispatcher heartbeatDispatcher;
 
   public SCMDatanodeProtocolServer(final OzoneConfiguration conf,
-      StorageContainerManager scm)  throws IOException {
+      StorageContainerManager scm, EventPublisher eventPublisher)
+      throws IOException {
 
     Preconditions.checkNotNull(scm, "SCM cannot be null");
+    Preconditions.checkNotNull(eventPublisher, "EventPublisher cannot be null");
+
     this.scm = scm;
     final int handlerCount =
         conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
             OZONE_SCM_HANDLER_COUNT_DEFAULT);
 
+    heartbeatDispatcher = new SCMDatanodeHeartbeatDispatcher(eventPublisher);
+
     RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
         ProtobufRpcEngine.class);
     BlockingService dnProtoPbService =
@@ -155,10 +160,6 @@ public class SCMDatanodeProtocolServer implements
             conf, OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr,
             datanodeRpcServer);
 
-    heartbeatDispatcher = SCMDatanodeHeartbeatDispatcher.newBuilder(conf, scm)
-        .addHandlerFor(NodeReportProto.class)
-        .addHandlerFor(ContainerReportsProto.class)
-        .build();
   }
 
   public void start() {
@@ -319,7 +320,6 @@ public class SCMDatanodeProtocolServer implements
     try {
       LOG.info("Stopping the RPC server for DataNodes");
       datanodeRpcServer.stop();
-      heartbeatDispatcher.shutdown();
     } catch (Exception ex) {
       LOG.error(" datanodeRpcServer stop failed.", ex);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 5725d23..568a86a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.common.StorageInfo;
-import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.StringUtils;
@@ -182,7 +181,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
       scmAdminUsernames.add(scmUsername);
     }
 
-    datanodeProtocolServer = new SCMDatanodeProtocolServer(conf, this);
+    datanodeProtocolServer = new SCMDatanodeProtocolServer(conf, this,
+        eventQueue);
     blockProtocolServer = new SCMBlockProtocolServer(conf, this);
     clientProtocolServer = new SCMClientProtocolServer(conf, this);
     httpServer = new StorageContainerManagerHttpServer(conf);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
deleted file mode 100644
index 00ce94d..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Handler for Datanode Container Report.
- */
-public class SCMDatanodeContainerReportHandler extends
-    SCMDatanodeReportHandler<ContainerReportsProto> {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      SCMDatanodeContainerReportHandler.class);
-
-  @Override
-  public void processReport(DatanodeDetails datanodeDetails,
-                            ContainerReportsProto report) throws IOException {
-    LOG.trace("Processing container report from {}.", datanodeDetails);
-    updateContainerReportMetrics(datanodeDetails, report);
-    getSCM().getScmContainerManager()
-        .processContainerReports(datanodeDetails, report);
-  }
-
-  /**
-   * Updates container report metrics in SCM.
-   *
-   * @param datanodeDetails Datanode Information
-   * @param reports Container Reports
-   */
-  private void updateContainerReportMetrics(DatanodeDetails datanodeDetails,
-                                            ContainerReportsProto reports) {
-    ContainerStat newStat = new ContainerStat();
-    for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports
-        .getReportsList()) {
-      newStat.add(new ContainerStat(info.getSize(), info.getUsed(),
-          info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(),
-          info.getReadCount(), info.getWriteCount()));
-    }
-    // update container metrics
-    StorageContainerManager.getMetrics().setLastContainerStat(newStat);
-
-    // Update container stat entry, this will trigger a removal operation if it
-    // exists in cache.
-    String datanodeUuid = datanodeDetails.getUuidString();
-    getSCM().getContainerReportCache().put(datanodeUuid, newStat);
-    // update global view container metrics
-    StorageContainerManager.getMetrics().incrContainerStat(newStat);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
deleted file mode 100644
index d50edff..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-/**
- * This class is responsible for dispatching heartbeat from datanode to
- * appropriate ReportHandlers at SCM.
- * Only one handler per report is supported now, it's very easy to support
- * multiple handlers for a report.
- */
-public final class SCMDatanodeHeartbeatDispatcher {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      SCMDatanodeHeartbeatDispatcher.class);
-
-  /**
-   * This stores Report to Handler mapping.
-   */
-  private final Map<Class<? extends GeneratedMessage>,
-      SCMDatanodeReportHandler<? extends GeneratedMessage>> handlers;
-
-  /**
-   * Executor service which will be used for processing reports.
-   */
-  private final ExecutorService executorService;
-
-  /**
-   * Constructs SCMDatanodeHeartbeatDispatcher instance with the given
-   * handlers.
-   *
-   * @param handlers report to report handler mapping
-   */
-  private SCMDatanodeHeartbeatDispatcher(Map<Class<? extends GeneratedMessage>,
-      SCMDatanodeReportHandler<? extends GeneratedMessage>> handlers) {
-    this.handlers = handlers;
-    this.executorService = HadoopExecutors.newCachedThreadPool(
-        new ThreadFactoryBuilder().setDaemon(true)
-            .setNameFormat("SCMDatanode Heartbeat Dispatcher Thread - %d")
-            .build());
-  }
-
-  /**
-   * Dispatches heartbeat to registered handlers.
-   *
-   * @param heartbeat heartbeat to be dispatched.
-   */
-  public void dispatch(SCMHeartbeatRequestProto heartbeat) {
-    DatanodeDetails datanodeDetails = DatanodeDetails
-        .getFromProtoBuf(heartbeat.getDatanodeDetails());
-    if (heartbeat.hasNodeReport()) {
-      processReport(datanodeDetails, heartbeat.getNodeReport());
-    }
-    if (heartbeat.hasContainerReport()) {
-      processReport(datanodeDetails, heartbeat.getContainerReport());
-    }
-  }
-
-  /**
-   * Invokes appropriate ReportHandler and submits the task to executor
-   * service for processing.
-   *
-   * @param datanodeDetails Datanode Information
-   * @param report Report to be processed
-   */
-  @SuppressWarnings("unchecked")
-  private void processReport(DatanodeDetails datanodeDetails,
-                             GeneratedMessage report) {
-    executorService.submit(() -> {
-      try {
-        SCMDatanodeReportHandler handler = handlers.get(report.getClass());
-        handler.processReport(datanodeDetails, report);
-      } catch (IOException ex) {
-        LOG.error("Exception wile processing report {}, from {}",
-            report.getClass(), datanodeDetails, ex);
-      }
-    });
-  }
-
-  /**
-   * Shuts down SCMDatanodeHeartbeatDispatcher.
-   */
-  public void shutdown() {
-    executorService.shutdown();
-  }
-
-  /**
-   * Returns a new Builder to construct {@link SCMDatanodeHeartbeatDispatcher}.
-   *
-   * @param conf Configuration to be used by SCMDatanodeHeartbeatDispatcher
-   * @param scm {@link StorageContainerManager} instance to be used by report
-   *            handlers
-   *
-   * @return {@link SCMDatanodeHeartbeatDispatcher.Builder} instance
-   */
-  public static Builder newBuilder(Configuration conf,
-                                   StorageContainerManager scm) {
-    return new Builder(conf, scm);
-  }
-
-  /**
-   * Builder for SCMDatanodeHeartbeatDispatcher.
-   */
-  public static class Builder {
-
-    private final SCMDatanodeReportHandlerFactory reportHandlerFactory;
-    private final Map<Class<? extends GeneratedMessage>,
-        SCMDatanodeReportHandler<? extends GeneratedMessage>> report2handler;
-
-    /**
-     * Constructs SCMDatanodeHeartbeatDispatcher.Builder instance.
-     *
-     * @param conf Configuration object to be used.
-     * @param scm StorageContainerManager instance to be used for report
-     *            handler initialization.
-     */
-    private Builder(Configuration conf, StorageContainerManager scm) {
-      this.report2handler = new HashMap<>();
-      this.reportHandlerFactory =
-          new SCMDatanodeReportHandlerFactory(conf, scm);
-    }
-
-    /**
-     * Adds new report handler for the given report.
-     *
-     * @param report Report for which handler has to be added
-     *
-     * @return Builder
-     */
-    public Builder addHandlerFor(Class<? extends GeneratedMessage> report) {
-      report2handler.put(report, reportHandlerFactory.getHandlerFor(report));
-      return this;
-    }
-
-    /**
-     * Associates the given report handler for the given report.
-     *
-     * @param report Report to be associated with
-     * @param handler Handler to be used for the report
-     *
-     * @return Builder
-     */
-    public Builder addHandler(Class<? extends GeneratedMessage> report,
-        SCMDatanodeReportHandler<? extends GeneratedMessage> handler) {
-      report2handler.put(report, handler);
-      return this;
-    }
-
-    /**
-     * Builds and returns {@link SCMDatanodeHeartbeatDispatcher} instance.
-     *
-     * @return SCMDatanodeHeartbeatDispatcher
-     */
-    public SCMDatanodeHeartbeatDispatcher build() {
-      return new SCMDatanodeHeartbeatDispatcher(report2handler);
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
deleted file mode 100644
index fb89b02..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Handles Datanode Node Report.
- */
-public class SCMDatanodeNodeReportHandler extends
-    SCMDatanodeReportHandler<NodeReportProto> {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      SCMDatanodeNodeReportHandler.class);
-
-  @Override
-  public void processReport(DatanodeDetails datanodeDetails,
-                            NodeReportProto report) throws IOException {
-    LOG.debug("Processing node report from {}.", datanodeDetails);
-    //TODO: add logic to process node report.
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
deleted file mode 100644
index d338649..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-
-import java.io.IOException;
-
-/**
- * Datanode Report handlers should implement this interface in order to get
- * call back whenever the report is received from datanode.
- *
- * @param <T> Type of report the handler is interested in.
- */
-public abstract class SCMDatanodeReportHandler<T extends GeneratedMessage>
-    implements Configurable {
-
-  private Configuration config;
-  private StorageContainerManager scm;
-
-  /**
-   * Initializes SCMDatanodeReportHandler and associates it with the given
-   * StorageContainerManager instance.
-   *
-   * @param storageContainerManager StorageContainerManager instance to be
-   *                                associated with.
-   */
-  public void init(StorageContainerManager storageContainerManager) {
-    this.scm = storageContainerManager;
-  }
-
-  /**
-   * Returns the associated StorageContainerManager instance. This will be
-   * used by the ReportHandler implementations.
-   *
-   * @return {@link StorageContainerManager}
-   */
-  protected StorageContainerManager getSCM() {
-    return scm;
-  }
-
-  @Override
-  public void setConf(Configuration conf) {
-    this.config = conf;
-  }
-
-  @Override
-  public Configuration getConf() {
-    return config;
-  }
-
-  /**
-   * Processes the report received from datanode. Each ReportHandler
-   * implementation is responsible for providing the logic to process the
-   * report it's interested in.
-   *
-   * @param datanodeDetails Datanode Information
-   * @param report Report to be processed
-   *
-   * @throws IOException In case of any exception
-   */
-  abstract void processReport(DatanodeDetails datanodeDetails, T report)
-      throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
deleted file mode 100644
index e88495f..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-/**
- * Factory class to construct {@link SCMDatanodeReportHandler} given a report.
- */
-public class SCMDatanodeReportHandlerFactory {
-
-  private final Configuration conf;
-  private final StorageContainerManager scm;
-  private final Map<Class<? extends GeneratedMessage>,
-      Class<? extends SCMDatanodeReportHandler<? extends GeneratedMessage>>>
-      report2handler;
-
-  /**
-   * Constructs {@link SCMDatanodeReportHandler} instance.
-   *
-   * @param conf Configuration to be passed to the
-   *               {@link SCMDatanodeReportHandler}
-   */
-  public SCMDatanodeReportHandlerFactory(Configuration conf,
-                                         StorageContainerManager scm) {
-    this.conf = conf;
-    this.scm = scm;
-    this.report2handler = new HashMap<>();
-
-    report2handler.put(NodeReportProto.class,
-        SCMDatanodeNodeReportHandler.class);
-    report2handler.put(ContainerReportsProto.class,
-        SCMDatanodeContainerReportHandler.class);
-  }
-
-  /**
-   * Returns the SCMDatanodeReportHandler for the corresponding report.
-   *
-   * @param report report
-   *
-   * @return report handler
-   */
-  public SCMDatanodeReportHandler<? extends GeneratedMessage> getHandlerFor(
-      Class<? extends GeneratedMessage> report) {
-    Class<? extends SCMDatanodeReportHandler<? extends GeneratedMessage>>
-        handlerClass = report2handler.get(report);
-    if (handlerClass == null) {
-      throw new RuntimeException("No handler found for report " + report);
-    }
-    SCMDatanodeReportHandler<? extends GeneratedMessage> instance =
-    ReflectionUtils.newInstance(handlerClass, conf);
-    instance.init(scm);
-    return instance;
-  }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
deleted file mode 100644
index fda3993..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.server.report;
-/**
- * Handling of all the datanode reports in SCM which are received through
- * heartbeat is done here.
- *
- * SCM Datanode Report Processing State Diagram:
- *
- *          SCMDatanode            SCMDatanodeHeartbeat    SCMDatanodeReport
- *         ProtocolServer              Dispatcher               Handler
- *                |                        |                        |
- *                |                        |                        |
- *                |    construct           |                        |
- *                |----------------------->|                        |
- *                |                        |                        |
- *                |                        |     register           |
- *                |                        |<-----------------------|
- *                |                        |                        |
- *   +------------+------------------------+------------------------+--------+
- *   | loop       |                        |                        |        |
- *   |            |                        |                        |        |
- *   |            |                        |                        |        |
- *   | heartbeat  |                        |                        |        |
- * - +----------->|                        |                        |        |
- *   |   from     |       heartbeat        |                        |        |
- *   | Datanode   |----------------------->|                        |        |
- *   |            |                        |       report           |        |
- *   |            |                        |----------------------->|        |
- *   |            |                        |                        |        |
- *   |   DN       |                        |                        |        |
- * <-+------------|                        |                        |        |
- *   | commands   |                        |                        |        |
- *   |            |                        |                        |        |
- *   +------------+------------------------+------------------------+--------+
- *                |                        |                        |
- *                |                        |                        |
- *                |     shutdown           |                        |
- *                |----------------------->|                        |
- *                |                        |                        |
- *                |                        |                        |
- *                -                        -                        -
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
new file mode 100644
index 0000000..326a34b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+    .ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+    .NodeReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the behavior of SCMDatanodeHeartbeatDispatcher.
+ */
+public class TestSCMDatanodeHeartbeatDispatcher {
+
+
+  @Test
+  public void testNodeReportDispatcher() throws IOException {
+
+    Configuration conf = new OzoneConfiguration();
+
+    AtomicInteger eventReceived = new AtomicInteger();
+
+    NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
+
+    SCMDatanodeHeartbeatDispatcher dispatcher =
+        new SCMDatanodeHeartbeatDispatcher(new EventPublisher() {
+          @Override
+          public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
+              EVENT_TYPE event, PAYLOAD payload) {
+            Assert.assertEquals(event,
+                SCMDatanodeHeartbeatDispatcher.NODE_REPORT);
+            eventReceived.incrementAndGet();
+            Assert.assertEquals(nodeReport, ((NodeReportFromDatanode)payload).getReport());
+
+          }
+        });
+
+    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+
+    SCMHeartbeatRequestProto heartbeat =
+        SCMHeartbeatRequestProto.newBuilder()
+        .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+        .setNodeReport(nodeReport)
+        .build();
+    dispatcher.dispatch(heartbeat);
+    Assert.assertEquals(1, eventReceived.get());
+
+
+  }
+
+  @Test
+  public void testContainerReportDispatcher() throws IOException {
+
+    Configuration conf = new OzoneConfiguration();
+
+    AtomicInteger eventReceived = new AtomicInteger();
+
+    ContainerReportsProto containerReport =
+        ContainerReportsProto.getDefaultInstance();
+
+    SCMDatanodeHeartbeatDispatcher dispatcher =
+        new SCMDatanodeHeartbeatDispatcher(new EventPublisher() {
+          @Override
+          public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
+              EVENT_TYPE event, PAYLOAD payload) {
+            Assert.assertEquals(event,
+                SCMDatanodeHeartbeatDispatcher.CONTAINER_REPORT);
+            Assert.assertEquals(containerReport, ((ContainerReportFromDatanode)payload).getReport());
+            eventReceived.incrementAndGet();
+          }
+        });
+
+    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+
+    SCMHeartbeatRequestProto heartbeat =
+        SCMHeartbeatRequestProto.newBuilder()
+            .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+            .setContainerReport(containerReport)
+            .build();
+    dispatcher.dispatch(heartbeat);
+    Assert.assertEquals(1, eventReceived.get());
+
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
deleted file mode 100644
index 776ae88..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify SCMDatanodeContainerReportHandler's behavior.
- */
-public class TestSCMDatanodeContainerReportHandler {
-
-  //TODO: add test cases to verify SCMDatanodeContainerReportHandler.
-
-  @Test
-  public void dummyTest() {
-    Assert.assertTrue(true);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
deleted file mode 100644
index 5d08647..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-/**
- * This class tests the behavior of SCMDatanodeHeartbeatDispatcher.
- */
-public class TestSCMDatanodeHeartbeatDispatcher {
-
-  @Test
-  public void testSCMDatanodeHeartbeatDispatcherBuilder() {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeHeartbeatDispatcher dispatcher =
-        SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
-        .addHandlerFor(NodeReportProto.class)
-        .addHandlerFor(ContainerReportsProto.class)
-        .build();
-    Assert.assertNotNull(dispatcher);
-  }
-
-  @Test
-  public void testNodeReportDispatcher() throws IOException {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeNodeReportHandler nodeReportHandler =
-        Mockito.mock(SCMDatanodeNodeReportHandler.class);
-    SCMDatanodeHeartbeatDispatcher dispatcher =
-        SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
-            .addHandler(NodeReportProto.class, nodeReportHandler)
-            .build();
-
-    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
-    NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
-    SCMHeartbeatRequestProto heartbeat =
-        SCMHeartbeatRequestProto.newBuilder()
-        .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
-        .setNodeReport(nodeReport)
-        .build();
-    dispatcher.dispatch(heartbeat);
-    verify(nodeReportHandler,
-        times(1))
-        .processReport(any(DatanodeDetails.class), eq(nodeReport));
-  }
-
-  @Test
-  public void testContainerReportDispatcher() throws IOException {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeContainerReportHandler containerReportHandler =
-        Mockito.mock(SCMDatanodeContainerReportHandler.class);
-    SCMDatanodeHeartbeatDispatcher dispatcher =
-        SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
-            .addHandler(ContainerReportsProto.class, containerReportHandler)
-            .build();
-
-    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
-    ContainerReportsProto containerReport =
-        ContainerReportsProto.getDefaultInstance();
-    SCMHeartbeatRequestProto heartbeat =
-        SCMHeartbeatRequestProto.newBuilder()
-            .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
-            .setContainerReport(containerReport)
-            .build();
-    dispatcher.dispatch(heartbeat);
-    verify(containerReportHandler,
-        times(1))
-        .processReport(any(DatanodeDetails.class),
-            any(ContainerReportsProto.class));
-  }
-
-  @Test
-  public void testNodeAndContainerReportDispatcher() throws IOException {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeNodeReportHandler nodeReportHandler =
-        Mockito.mock(SCMDatanodeNodeReportHandler.class);
-    SCMDatanodeContainerReportHandler containerReportHandler =
-        Mockito.mock(SCMDatanodeContainerReportHandler.class);
-    SCMDatanodeHeartbeatDispatcher dispatcher =
-        SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
-            .addHandler(NodeReportProto.class, nodeReportHandler)
-            .addHandler(ContainerReportsProto.class, containerReportHandler)
-            .build();
-
-    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
-    NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
-    ContainerReportsProto containerReport =
-        ContainerReportsProto.getDefaultInstance();
-    SCMHeartbeatRequestProto heartbeat =
-        SCMHeartbeatRequestProto.newBuilder()
-            .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
-            .setNodeReport(nodeReport)
-            .setContainerReport(containerReport)
-            .build();
-    dispatcher.dispatch(heartbeat);
-    verify(nodeReportHandler,
-        times(1))
-        .processReport(any(DatanodeDetails.class), any(NodeReportProto.class));
-    verify(containerReportHandler,
-        times(1))
-        .processReport(any(DatanodeDetails.class),
-            any(ContainerReportsProto.class));
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
deleted file mode 100644
index 30a753c..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify TestSCMDatanodeNodeReportHandler's behavior.
- */
-public class TestSCMDatanodeNodeReportHandler {
-
-
-  //TODO: add test cases to verify SCMDatanodeNodeReportHandler.
-
-  @Test
-  public void dummyTest() {
-    Assert.assertTrue(true);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
deleted file mode 100644
index 4b918f7..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify the functionality of SCMDatanodeReportHandlerFactory.
- */
-public class TestSCMDatanodeReportHandlerFactory {
-
-  @Test
-  public void testNodeReportHandlerConstruction() {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeReportHandlerFactory factory =
-        new SCMDatanodeReportHandlerFactory(conf, null);
-    Assert.assertTrue(factory.getHandlerFor(NodeReportProto.class)
-        instanceof SCMDatanodeNodeReportHandler);
-  }
-
-  @Test
-  public void testContainerReporttHandlerConstruction() {
-    Configuration conf = new OzoneConfiguration();
-    SCMDatanodeReportHandlerFactory factory =
-        new SCMDatanodeReportHandlerFactory(conf, null);
-    Assert.assertTrue(factory.getHandlerFor(ContainerReportsProto.class)
-        instanceof SCMDatanodeContainerReportHandler);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
deleted file mode 100644
index 4a3f59f..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-/**
- * Contains test-cases to test Datanode report handlers in SCM.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
deleted file mode 100644
index ecddf8e..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.server.report
-    .SCMDatanodeContainerReportHandler;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerReport;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
-import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-/**
- * This class tests the metrics of Storage Container Manager.
- */
-public class TestSCMMetrics {
-  /**
-   * Set the timeout for each test.
-   */
-  @Rule
-  public Timeout testTimeout = new Timeout(90000);
-
-  private static MiniOzoneCluster cluster = null;
-
-  @Test
-  public void testContainerMetrics() throws Exception {
-    int nodeCount = 2;
-    int numReport = 2;
-    long size = OzoneConsts.GB * 5;
-    long used = OzoneConsts.GB * 2;
-    long readBytes = OzoneConsts.GB * 1;
-    long writeBytes = OzoneConsts.GB * 2;
-    int keyCount = 1000;
-    int readCount = 100;
-    int writeCount = 50;
-    OzoneConfiguration conf = new OzoneConfiguration();
-
-    try {
-      cluster = MiniOzoneCluster.newBuilder(conf)
-          .setNumDatanodes(nodeCount).build();
-      cluster.waitForClusterToBeReady();
-
-      ContainerStat stat = new ContainerStat(size, used, keyCount, readBytes,
-          writeBytes, readCount, writeCount);
-      StorageContainerManager scmManager = cluster.getStorageContainerManager();
-      DatanodeDetails fstDatanodeDetails = TestUtils.getDatanodeDetails();
-      ContainerReportsProto request = createContainerReport(numReport, stat);
-      String fstDatanodeUuid = fstDatanodeDetails.getUuidString();
-      SCMDatanodeContainerReportHandler containerReportHandler =
-          new SCMDatanodeContainerReportHandler();
-      containerReportHandler.setConf(conf);
-      containerReportHandler.init(scmManager);
-      containerReportHandler.processReport(
-          fstDatanodeDetails, request);
-
-      // verify container stat metrics
-      MetricsRecordBuilder scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
-      assertEquals(size * numReport,
-          getLongGauge("LastContainerReportSize", scmMetrics));
-      assertEquals(used * numReport,
-          getLongGauge("LastContainerReportUsed", scmMetrics));
-      assertEquals(readBytes * numReport,
-          getLongGauge("LastContainerReportReadBytes", scmMetrics));
-      assertEquals(writeBytes * numReport,
-          getLongGauge("LastContainerReportWriteBytes", scmMetrics));
-
-      assertEquals(keyCount * numReport,
-          getLongGauge("LastContainerReportKeyCount", scmMetrics));
-      assertEquals(readCount * numReport,
-          getLongGauge("LastContainerReportReadCount", scmMetrics));
-      assertEquals(writeCount * numReport,
-          getLongGauge("LastContainerReportWriteCount", scmMetrics));
-
-      // add one new report
-      DatanodeDetails sndDatanodeDetails = TestUtils.getDatanodeDetails();
-      request = createContainerReport(1, stat);
-      String sndDatanodeUuid = sndDatanodeDetails.getUuidString();
-      containerReportHandler.processReport(
-          sndDatanodeDetails, request);
-
-      scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
-      assertEquals(size * (numReport + 1),
-          getLongCounter("ContainerReportSize", scmMetrics));
-      assertEquals(used * (numReport + 1),
-          getLongCounter("ContainerReportUsed", scmMetrics));
-      assertEquals(readBytes * (numReport + 1),
-          getLongCounter("ContainerReportReadBytes", scmMetrics));
-      assertEquals(writeBytes * (numReport + 1),
-          getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
-      assertEquals(keyCount * (numReport + 1),
-          getLongCounter("ContainerReportKeyCount", scmMetrics));
-      assertEquals(readCount * (numReport + 1),
-          getLongCounter("ContainerReportReadCount", scmMetrics));
-      assertEquals(writeCount * (numReport + 1),
-          getLongCounter("ContainerReportWriteCount", scmMetrics));
-
-      // Re-send reports but with different value for validating
-      // the aggregation.
-      stat = new ContainerStat(100, 50, 3, 50, 60, 5, 6);
-      containerReportHandler.processReport(
-          fstDatanodeDetails, createContainerReport(1, stat));
-
-      stat = new ContainerStat(1, 1, 1, 1, 1, 1, 1);
-      containerReportHandler.processReport(
-          sndDatanodeDetails, createContainerReport(1, stat));
-
-      // the global container metrics value should be updated
-      scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
-      assertEquals(101, getLongCounter("ContainerReportSize", scmMetrics));
-      assertEquals(51, getLongCounter("ContainerReportUsed", scmMetrics));
-      assertEquals(51, getLongCounter("ContainerReportReadBytes", scmMetrics));
-      assertEquals(61, getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
-      assertEquals(4, getLongCounter("ContainerReportKeyCount", scmMetrics));
-      assertEquals(6, getLongCounter("ContainerReportReadCount", scmMetrics));
-      assertEquals(7, getLongCounter("ContainerReportWriteCount", scmMetrics));
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  @Test
-  public void testStaleNodeContainerReport() throws Exception {
-    int nodeCount = 2;
-    int numReport = 2;
-    long size = OzoneConsts.GB * 5;
-    long used = OzoneConsts.GB * 2;
-    long readBytes = OzoneConsts.GB * 1;
-    long writeBytes = OzoneConsts.GB * 2;
-    int keyCount = 1000;
-    int readCount = 100;
-    int writeCount = 50;
-    OzoneConfiguration conf = new OzoneConfiguration();
-
-    try {
-      cluster = MiniOzoneCluster.newBuilder(conf)
-          .setNumDatanodes(nodeCount).build();
-      cluster.waitForClusterToBeReady();
-
-      ContainerStat stat = new ContainerStat(size, used, keyCount, readBytes,
-          writeBytes, readCount, writeCount);
-      StorageContainerManager scmManager = cluster.getStorageContainerManager();
-
-      DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
-          .getDatanodeDetails();
-      SCMDatanodeContainerReportHandler containerReportHandler =
-          new SCMDatanodeContainerReportHandler();
-      containerReportHandler.setConf(conf);
-      containerReportHandler.init(scmManager);
-      ContainerReportsProto request = createContainerReport(numReport, stat);
-      containerReportHandler.processReport(
-          datanodeDetails, request);
-
-      MetricsRecordBuilder scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
-      assertEquals(size * numReport,
-          getLongCounter("ContainerReportSize", scmMetrics));
-      assertEquals(used * numReport,
-          getLongCounter("ContainerReportUsed", scmMetrics));
-      assertEquals(readBytes * numReport,
-          getLongCounter("ContainerReportReadBytes", scmMetrics));
-      assertEquals(writeBytes * numReport,
-          getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
-      assertEquals(keyCount * numReport,
-          getLongCounter("ContainerReportKeyCount", scmMetrics));
-      assertEquals(readCount * numReport,
-          getLongCounter("ContainerReportReadCount", scmMetrics));
-      assertEquals(writeCount * numReport,
-          getLongCounter("ContainerReportWriteCount", scmMetrics));
-
-      // reset stale interval time to move node from healthy to stale
-      SCMNodeManager nodeManager = (SCMNodeManager) cluster
-          .getStorageContainerManager().getScmNodeManager();
-      nodeManager.setStaleNodeIntervalMs(100);
-
-      // verify the metrics when node becomes stale
-      GenericTestUtils.waitFor(() -> {
-        MetricsRecordBuilder metrics = getMetrics(SCMMetrics.SOURCE_NAME);
-        return 0 == getLongCounter("ContainerReportSize", metrics)
-            && 0 == getLongCounter("ContainerReportUsed", metrics)
-            && 0 == getLongCounter("ContainerReportReadBytes", metrics)
-            && 0 == getLongCounter("ContainerReportWriteBytes", metrics)
-            && 0 == getLongCounter("ContainerReportKeyCount", metrics)
-            && 0 == getLongCounter("ContainerReportReadCount", metrics)
-            && 0 == getLongCounter("ContainerReportWriteCount", metrics);
-      }, 1000, 60000);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
-
-  private ContainerReportsProto createContainerReport(int numReport,
-      ContainerStat stat) {
-    StorageContainerDatanodeProtocolProtos.ContainerReportsProto.Builder
-        reportsBuilder = StorageContainerDatanodeProtocolProtos
-        .ContainerReportsProto.newBuilder();
-
-    for (int i = 0; i < numReport; i++) {
-      ContainerReport report = new ContainerReport(
-          RandomUtils.nextLong(), DigestUtils.sha256Hex("Simulated"));
-      report.setSize(stat.getSize().get());
-      report.setBytesUsed(stat.getUsed().get());
-      report.setReadCount(stat.getReadCount().get());
-      report.setReadBytes(stat.getReadBytes().get());
-      report.setKeyCount(stat.getKeyCount().get());
-      report.setWriteCount(stat.getWriteCount().get());
-      report.setWriteBytes(stat.getWriteBytes().get());
-      reportsBuilder.addReports(report.getProtoBufMessage());
-    }
-    return reportsBuilder.build();
-  }
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[32/50] [abbrv] hadoop git commit: HDDS-94. Change ozone datanode command to start the standalone datanode plugin. Contributed by Sandeep Nemuri.

Posted by gi...@apache.org.
HDDS-94. Change ozone datanode command to start the standalone datanode plugin.
Contributed by  Sandeep Nemuri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18932717
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18932717
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18932717

Branch: refs/heads/HADOOP-15461
Commit: 18932717c42382ed8842de7719ec6d20c1765366
Parents: 1e30547
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Jun 26 18:28:47 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 14:15:15 2018 -0700

----------------------------------------------------------------------
 hadoop-dist/src/main/compose/ozone/docker-compose.yaml | 12 ------------
 hadoop-dist/src/main/compose/ozone/docker-config       |  5 -----
 .../src/main/compose/ozoneperf/docker-compose.yaml     | 13 -------------
 hadoop-dist/src/main/compose/ozoneperf/docker-config   |  5 -----
 .../org/apache/hadoop/ozone/HddsDatanodeService.java   | 13 ++++++++++++-
 .../src/test/acceptance/basic/docker-compose.yaml      | 12 ------------
 .../src/test/acceptance/basic/docker-config            |  5 -----
 .../src/test/acceptance/commonlib.robot                |  1 -
 .../src/test/acceptance/ozonefs/docker-compose.yaml    | 12 ------------
 .../src/test/acceptance/ozonefs/docker-config          |  5 -----
 hadoop-ozone/common/src/main/bin/ozone                 |  9 +++------
 11 files changed, 15 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
index faf420c..512c649 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ../../ozone:/opt/hadoop
-      ports:
-         - 9870:9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
index c693db0..632f870 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
index fb7873b..3233c11 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
@@ -16,19 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ../../ozone:/opt/hadoop
-         - ./jmxpromo.jar:/opt/jmxpromo.jar
-      ports:
-         - 9870:9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config
index e4f5485..2be22a7 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.scm.names=scm
 OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index fa4187a..ddeec87 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,9 +25,11 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.statemachine
     .DatanodeStateMachine;
+import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.ServicePlugin;
 import org.apache.hadoop.util.StringUtils;
 import org.slf4j.Logger;
@@ -229,9 +231,18 @@ public class HddsDatanodeService implements ServicePlugin {
 
   public static void main(String[] args) {
     try {
+      if (DFSUtil.parseHelpArgument(args, "Starts HDDS Datanode", System.out, false)) {
+        System.exit(0);
+      }
+      Configuration conf = new OzoneConfiguration();
+      GenericOptionsParser hParser = new GenericOptionsParser(conf, args);
+      if (!hParser.isParseSuccessful()) {
+        GenericOptionsParser.printGenericCommandUsage(System.err);
+        System.exit(1);
+      }
       StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG);
       HddsDatanodeService hddsDatanodeService =
-          createHddsDatanodeService(new OzoneConfiguration());
+          createHddsDatanodeService(conf);
       hddsDatanodeService.start(null);
       hddsDatanodeService.join();
     } catch (Throwable e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
index 44bd4a0..b50f42d 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
index 180dc8e..c3ec2ef 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
 OZONE-SITE.XML_ozone.scm.names=scm
@@ -24,13 +23,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
 OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
index 01ed302..a5ea30a 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
@@ -28,7 +28,6 @@ Startup Ozone cluster with size
 Daemons are running without error
     Is daemon running without error           ksm
     Is daemon running without error           scm
-    Is daemon running without error           namenode
     Is daemon running without error           datanode
 
 Check if datanode is connected to the scm

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
index 3323557..12022df 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -16,18 +16,6 @@
 
 version: "3"
 services:
-   namenode:
-      image: apache/hadoop-runner
-      hostname: namenode
-      volumes:
-         - ${OZONEDIR}:/opt/hadoop
-      ports:
-         - 9870
-      environment:
-          ENSURE_NAMENODE_DIR: /data/namenode
-      env_file:
-         - ./docker-config
-      command: ["/opt/hadoop/bin/hdfs","namenode"]
    datanode:
       image: apache/hadoop-runner
       volumes:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
index dec863e..e06d434 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
 CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
 OZONE-SITE.XML_ozone.ksm.address=ksm
 OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
@@ -25,13 +24,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
 OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
 OZONE-SITE.XML_ozone.handler.type=distributed
 OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
 OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
 HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 6843bdd..390f089 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -34,7 +34,7 @@ function hadoop_usage
 
 
   hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
-  hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
+  hadoop_add_subcommand "datanode" daemon "run a HDDS datanode"
   hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
   hadoop_add_subcommand "freon" client "runs an ozone data generator"
   hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
@@ -45,7 +45,7 @@ function hadoop_usage
   hadoop_add_subcommand "o3" client "command line interface for ozone"
   hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
   hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
-  hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
+  hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
 
@@ -68,10 +68,7 @@ function ozonecmd_case
     ;;
     datanode)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
-      HADOOP_SECURE_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
-      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
-      hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR
-      hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR
+      HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
     ;;
     envvars)
       echo "JAVA_HOME='${JAVA_HOME}'"


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[08/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable ozonefs artifact Contributed by Marton, Elek.

Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
new file mode 100644
index 0000000..3156eb2
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract test for ROOT directory operations.
+ */
+public class ITestOzoneContractRootDir extends
+    AbstractContractRootDirectoryTest {
+
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
new file mode 100644
index 0000000..c4bc0ff
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering file seek.
+ */
+public class ITestOzoneContractSeek extends AbstractContractSeekTest {
+  @BeforeClass
+  public static void createCluster() throws IOException {
+    OzoneContract.createCluster();
+  }
+
+  @AfterClass
+  public static void teardownCluster() throws IOException {
+    OzoneContract.destroyCluster();
+  }
+
+  @Override
+  protected AbstractFSContract createContract(Configuration conf) {
+    return new OzoneContract(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
new file mode 100644
index 0000000..8417e46
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+/**
+ * The contract of Ozone: only enabled if the test bucket is provided.
+ */
+class OzoneContract extends AbstractFSContract {
+
+  private static MiniOzoneCluster cluster;
+  private static StorageHandler storageHandler;
+  private static final String CONTRACT_XML = "contract/ozone.xml";
+
+  OzoneContract(Configuration conf) {
+    super(conf);
+    //insert the base features
+    addConfResource(CONTRACT_XML);
+  }
+
+  @Override
+  public String getScheme() {
+    return OzoneConsts.OZONE_URI_SCHEME;
+  }
+
+  @Override
+  public Path getTestPath() {
+    Path path = new Path("/test");
+    return path;
+  }
+
+  public static void createCluster() throws IOException {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.addResource(CONTRACT_XML);
+
+    cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+    try {
+      cluster.waitForClusterToBeReady();
+    } catch (Exception e) {
+      throw new IOException(e);
+    }
+    storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
+  }
+
+  private void copyClusterConfigs(String configKey) {
+    getConf().set(configKey, cluster.getConf().get(configKey));
+  }
+
+  @Override
+  public FileSystem getTestFileSystem() throws IOException {
+    //assumes cluster is not null
+    Assert.assertNotNull("cluster not created", cluster);
+
+    String userName = "user" + RandomStringUtils.randomNumeric(5);
+    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+
+
+    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+        null, null, null, null);
+    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+    volumeArgs.setUserName(userName);
+    volumeArgs.setAdminName(adminName);
+    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+    try {
+      storageHandler.createVolume(volumeArgs);
+      storageHandler.createBucket(bucketArgs);
+    } catch (OzoneException e) {
+      throw new IOException(e.getMessage());
+    }
+
+    String uri = String.format("%s://%s.%s/",
+        OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+    getConf().set("fs.defaultFS", uri);
+    copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
+    copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
+    return FileSystem.get(getConf());
+  }
+
+  public static void destroyCluster() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
new file mode 100644
index 0000000..fe2075c
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
@@ -0,0 +1,113 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<configuration>
+  <!--
+  Ozone is a blobstore, with very different behavior than a classic filesystem.
+  -->
+
+    <property>
+        <name>fs.contract.test.root-tests-enabled</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.test.random-seek-count</name>
+        <value>10</value>
+    </property>
+
+    <property>
+        <name>fs.contract.is-blobstore</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.create-visibility-delayed</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.is-case-sensitive</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.rename-returns-false-if-source-missing</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-append</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-atomic-directory-delete</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-atomic-rename</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-block-locality</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-concat</name>
+        <value>false</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-getfilestatus</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-seek</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-seek-on-closed-file</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.rejects-seek-past-eof</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-strict-exceptions</name>
+        <value>true</value>
+    </property>
+
+    <property>
+        <name>fs.contract.supports-unix-permissions</name>
+        <value>false</value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bf1619
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+
+# for debugging low level Ozone operations, uncomment this line
+# log4j.logger.org.apache.hadoop.ozone=DEBUG

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index cffef14..b655088 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <module>client</module>
     <module>ozone-manager</module>
     <module>tools</module>
+    <module>ozonefs</module>
     <module>integration-test</module>
     <module>objectstore-service</module>
     <module>docs</module>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ed0187b..dfd1eac 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -586,7 +586,11 @@
         <artifactId>hadoop-ozone-docs</artifactId>
         <version>${hdds.version}</version>
       </dependency>
-
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-ozone-filesystem</artifactId>
+        <version>${hdds.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdds-common</artifactId>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/pom.xml b/hadoop-tools/hadoop-ozone/pom.xml
deleted file mode 100644
index a7d0cfa..0000000
--- a/hadoop-tools/hadoop-ozone/pom.xml
+++ /dev/null
@@ -1,174 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-project</artifactId>
-    <version>3.2.0-SNAPSHOT</version>
-    <relativePath>../../hadoop-project</relativePath>
-  </parent>
-  <artifactId>hadoop-ozone-filesystem</artifactId>
-  <name>Apache Hadoop Ozone FileSystem</name>
-  <packaging>jar</packaging>
-
-  <properties>
-    <file.encoding>UTF-8</file.encoding>
-    <downloadSources>true</downloadSources>
-  </properties>
-
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-        <executions>
-          <execution>
-            <goals>
-              <goal>test-jar</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>deplist</id>
-            <phase>compile</phase>
-            <goals>
-              <goal>list</goal>
-            </goals>
-            <configuration>
-              <!-- build a shellprofile -->
-              <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs-client</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-server-scm</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-server-framework</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-ozone-ozone-manager</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-container-service</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-client</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-ozone-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-ozone-objectstore-service</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-ozone-client</artifactId>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-ozone-integration-test</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-distcp</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-distcp</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-</project>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
deleted file mode 100644
index 832a0cb..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-/**
- * Constants for Ozone FileSystem implementation.
- */
-public final class Constants {
-
-  public static final String OZONE_DEFAULT_USER = "hdfs";
-
-  public static final String OZONE_USER_DIR = "/user";
-
-  /** Local buffer directory. */
-  public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
-
-  /** Temporary directory. */
-  public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
-
-  /** Page size for Ozone listing operation. */
-  public static final int LISTING_PAGE_SIZE = 1024;
-
-  private Constants() {
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
deleted file mode 100644
index 4163c13..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * ozone implementation of AbstractFileSystem.
- * This impl delegates to the OzoneFileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class OzFs extends DelegateToFileSystem {
-
-  public OzFs(URI theUri, Configuration conf)
-      throws IOException, URISyntaxException {
-    super(theUri, new OzoneFileSystem(), conf,
-        OzoneConsts.OZONE_URI_SCHEME, false);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
deleted file mode 100644
index 4c5c0c8..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * The input stream for Ozone file system.
- *
- * TODO: Make inputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class OzoneFSInputStream extends FSInputStream {
-
-  private final ChunkGroupInputStream inputStream;
-
-  public OzoneFSInputStream(InputStream inputStream) {
-    this.inputStream = (ChunkGroupInputStream)inputStream;
-  }
-
-  @Override
-  public int read() throws IOException {
-    return inputStream.read();
-  }
-
-  @Override
-  public int read(byte[] b, int off, int len) throws IOException {
-    return inputStream.read(b, off, len);
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    inputStream.close();
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    inputStream.seek(pos);
-  }
-
-  @Override
-  public long getPos() throws IOException {
-    return inputStream.getPos();
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-
-  @Override
-  public int available() throws IOException {
-    return inputStream.available();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
deleted file mode 100644
index faa3628..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-
-
-/**
- * The output stream for Ozone file system.
- *
- * TODO: Make outputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-public class OzoneFSOutputStream extends OutputStream {
-
-  private final ChunkGroupOutputStream outputStream;
-
-  public OzoneFSOutputStream(OutputStream outputStream) {
-    this.outputStream = (ChunkGroupOutputStream)outputStream;
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    outputStream.write(b);
-  }
-
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    outputStream.write(b, off, len);
-  }
-
-  @Override
-  public synchronized void flush() throws IOException {
-    outputStream.flush();
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    outputStream.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
deleted file mode 100644
index 6906a9d..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Iterator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
-
-/**
- * The Ozone Filesystem implementation.
- *
- * This subclass is marked as private as code should not be creating it
- * directly; use {@link FileSystem#get(Configuration)} and variants to create
- * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
- * accessed. Consider those private and unstable.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class OzoneFileSystem extends FileSystem {
-  static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
-
-  /** The Ozone client for connecting to Ozone server. */
-  private OzoneClient ozoneClient;
-  private ObjectStore objectStore;
-  private OzoneVolume volume;
-  private OzoneBucket bucket;
-  private URI uri;
-  private String userName;
-  private Path workingDir;
-  private ReplicationType replicationType;
-  private ReplicationFactor replicationFactor;
-
-  private static final Pattern URL_SCHEMA_PATTERN =
-      Pattern.compile("(.+)\\.([^\\.]+)");
-
-  @Override
-  public void initialize(URI name, Configuration conf) throws IOException {
-    super.initialize(name, conf);
-    setConf(conf);
-    Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
-    assert getScheme().equals(name.getScheme());
-
-    String authority = name.getAuthority();
-
-    Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
-
-    if (!matcher.matches()) {
-      throw new IllegalArgumentException("Ozone file system url should be "
-          + "in the form o3://bucket.volume");
-    }
-    String bucketStr = matcher.group(1);
-    String volumeStr = matcher.group(2);
-
-    try {
-      uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
-          .setHost(authority).build();
-      LOG.trace("Ozone URI for ozfs initialization is " + uri);
-      this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
-      objectStore = ozoneClient.getObjectStore();
-      this.volume = objectStore.getVolume(volumeStr);
-      this.bucket = volume.getBucket(bucketStr);
-      this.replicationType = ReplicationType.valueOf(
-          conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
-              OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
-      this.replicationFactor = ReplicationFactor.valueOf(
-          conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
-              OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
-      try {
-        this.userName =
-            UserGroupInformation.getCurrentUser().getShortUserName();
-      } catch (IOException e) {
-        this.userName = OZONE_DEFAULT_USER;
-      }
-      this.workingDir = new Path(OZONE_USER_DIR, this.userName)
-              .makeQualified(this.uri, this.workingDir);
-    } catch (URISyntaxException ue) {
-      final String msg = "Invalid Ozone endpoint " + name;
-      LOG.error(msg, ue);
-      throw new IOException(msg, ue);
-    }
-  }
-
-  @Override
-  public void close() throws IOException {
-    try {
-      ozoneClient.close();
-    } finally {
-      super.close();
-    }
-  }
-
-  @Override
-  public URI getUri() {
-    return uri;
-  }
-
-  @Override
-  public String getScheme() {
-    return OZONE_URI_SCHEME;
-  }
-
-  @Override
-  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
-    LOG.trace("open() path:{}", f);
-    final FileStatus fileStatus = getFileStatus(f);
-    final String key = pathToKey(f);
-    if (fileStatus.isDirectory()) {
-      throw new FileNotFoundException("Can't open directory " + f + " to read");
-    }
-
-    return new FSDataInputStream(
-        new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
-  }
-
-  @Override
-  public FSDataOutputStream create(Path f, FsPermission permission,
-                                   boolean overwrite, int bufferSize,
-                                   short replication, long blockSize,
-                                   Progressable progress) throws IOException {
-    LOG.trace("create() path:{}", f);
-    final String key = pathToKey(f);
-    final FileStatus status;
-    try {
-      status = getFileStatus(f);
-      if (status.isDirectory()) {
-        throw new FileAlreadyExistsException(f + " is a directory");
-      } else {
-        if (!overwrite) {
-          // path references a file and overwrite is disabled
-          throw new FileAlreadyExistsException(f + " already exists");
-        }
-        LOG.trace("Overwriting file {}", f);
-        deleteObject(key);
-      }
-    } catch (FileNotFoundException ignored) {
-      // check if the parent directory needs to be created
-      Path parent = f.getParent();
-      try {
-        // create all the directories for the parent
-        FileStatus parentStatus = getFileStatus(parent);
-        LOG.trace("parent key:{} status:{}", key, parentStatus);
-      } catch (FileNotFoundException e) {
-        mkdirs(parent);
-      }
-      // This exception needs to ignored as this means that the file currently
-      // does not exists and a new file can thus be created.
-    }
-
-    OzoneOutputStream ozoneOutputStream =
-        bucket.createKey(key, 0, replicationType, replicationFactor);
-    // We pass null to FSDataOutputStream so it won't count writes that
-    // are being buffered to a file
-    return new FSDataOutputStream(
-        new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
-  }
-
-  @Override
-  public FSDataOutputStream createNonRecursive(Path path,
-      FsPermission permission,
-      EnumSet<CreateFlag> flags,
-      int bufferSize,
-      short replication,
-      long blockSize,
-      Progressable progress) throws IOException {
-    final Path parent = path.getParent();
-    if (parent != null) {
-      // expect this to raise an exception if there is no parent
-      if (!getFileStatus(parent).isDirectory()) {
-        throw new FileAlreadyExistsException("Not a directory: " + parent);
-      }
-    }
-    return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
-        bufferSize, replication, blockSize, progress);
-  }
-
-  @Override
-  public FSDataOutputStream append(Path f, int bufferSize,
-      Progressable progress) throws IOException {
-    throw new UnsupportedOperationException("append() Not implemented by the "
-        + getClass().getSimpleName() + " FileSystem implementation");
-  }
-
-  private class RenameIterator extends OzoneListingIterator {
-    private final String srcKey;
-    private final String dstKey;
-
-    RenameIterator(Path srcPath, Path dstPath)
-        throws IOException {
-      super(srcPath);
-      srcKey = pathToKey(srcPath);
-      dstKey = pathToKey(dstPath);
-      LOG.trace("rename from:{} to:{}", srcKey, dstKey);
-    }
-
-    boolean processKey(String key) throws IOException {
-      String newKeyName = dstKey.concat(key.substring(srcKey.length()));
-      bucket.renameKey(key, newKeyName);
-      return true;
-    }
-  }
-
-  /**
-   * Check whether the source and destination path are valid and then perform
-   * rename from source path to destination path.
-   *
-   * The rename operation is performed by renaming the keys with src as prefix.
-   * For such keys the prefix is changed from src to dst.
-   *
-   * @param src source path for rename
-   * @param dst destination path for rename
-   * @return true if rename operation succeeded or
-   * if the src and dst have the same path and are of the same type
-   * @throws IOException on I/O errors or if the src/dst paths are invalid.
-   */
-  @Override
-  public boolean rename(Path src, Path dst) throws IOException {
-    if (src.equals(dst)) {
-      return true;
-    }
-
-    LOG.trace("rename() from:{} to:{}", src, dst);
-    if (src.isRoot()) {
-      // Cannot rename root of file system
-      LOG.trace("Cannot rename the root of a filesystem");
-      return false;
-    }
-
-    // Cannot rename a directory to its own subdirectory
-    Path dstParent = dst.getParent();
-    while (dstParent != null && !src.equals(dstParent)) {
-      dstParent = dstParent.getParent();
-    }
-    Preconditions.checkArgument(dstParent == null,
-        "Cannot rename a directory to its own subdirectory");
-    // Check if the source exists
-    FileStatus srcStatus;
-    try {
-      srcStatus = getFileStatus(src);
-    } catch (FileNotFoundException fnfe) {
-      // source doesn't exist, return
-      return false;
-    }
-
-    // Check if the destination exists
-    FileStatus dstStatus;
-    try {
-      dstStatus = getFileStatus(dst);
-    } catch (FileNotFoundException fnde) {
-      dstStatus = null;
-    }
-
-    if (dstStatus == null) {
-      // If dst doesn't exist, check whether dst parent dir exists or not
-      // if the parent exists, the source can still be renamed to dst path
-      dstStatus = getFileStatus(dst.getParent());
-      if (!dstStatus.isDirectory()) {
-        throw new IOException(String.format(
-            "Failed to rename %s to %s, %s is a file", src, dst,
-            dst.getParent()));
-      }
-    } else {
-      // if dst exists and source and destination are same,
-      // check both the src and dst are of same type
-      if (srcStatus.getPath().equals(dstStatus.getPath())) {
-        return !srcStatus.isDirectory();
-      } else if (dstStatus.isDirectory()) {
-        // If dst is a directory, rename source as subpath of it.
-        // for example rename /source to /dst will lead to /dst/source
-        dst = new Path(dst, src.getName());
-        FileStatus[] statuses;
-        try {
-          statuses = listStatus(dst);
-        } catch (FileNotFoundException fnde) {
-          statuses = null;
-        }
-
-        if (statuses != null && statuses.length > 0) {
-          // If dst exists and not a directory not empty
-          throw new FileAlreadyExistsException(String.format(
-              "Failed to rename %s to %s, file already exists or not empty!",
-              src, dst));
-        }
-      } else {
-        // If dst is not a directory
-        throw new FileAlreadyExistsException(String.format(
-            "Failed to rename %s to %s, file already exists!", src, dst));
-      }
-    }
-
-    if (srcStatus.isDirectory()) {
-      if (dst.toString().startsWith(src.toString())) {
-        LOG.trace("Cannot rename a directory to a subdirectory of self");
-        return false;
-      }
-    }
-    RenameIterator iterator = new RenameIterator(src, dst);
-    return iterator.iterate();
-  }
-
-  private class DeleteIterator extends OzoneListingIterator {
-    private boolean recursive;
-    DeleteIterator(Path f, boolean recursive)
-        throws IOException {
-      super(f);
-      this.recursive = recursive;
-      if (getStatus().isDirectory()
-          && !this.recursive
-          && listStatus(f).length != 0) {
-        throw new PathIsNotEmptyDirectoryException(f.toString());
-      }
-    }
-
-    boolean processKey(String key) throws IOException {
-      if (key.equals("")) {
-        LOG.trace("Skipping deleting root directory");
-        return true;
-      } else {
-        LOG.trace("deleting key:" + key);
-        boolean succeed = deleteObject(key);
-        // if recursive delete is requested ignore the return value of
-        // deleteObject and issue deletes for other keys.
-        return recursive || succeed;
-      }
-    }
-  }
-
-  @Override
-  public boolean delete(Path f, boolean recursive) throws IOException {
-    LOG.trace("delete() path:{} recursive:{}", f, recursive);
-    try {
-      DeleteIterator iterator = new DeleteIterator(f, recursive);
-      return iterator.iterate();
-    } catch (FileNotFoundException e) {
-      LOG.debug("Couldn't delete {} - does not exist", f);
-      return false;
-    }
-  }
-
-  private class ListStatusIterator extends OzoneListingIterator {
-    private  List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
-    private Path f;
-
-    ListStatusIterator(Path f) throws IOException  {
-      super(f);
-      this.f = f;
-    }
-
-    boolean processKey(String key) throws IOException {
-      Path keyPath = new Path(OZONE_URI_DELIMITER + key);
-      if (key.equals(getPathKey())) {
-        if (pathIsDirectory()) {
-          return true;
-        } else {
-          statuses.add(getFileStatus(keyPath));
-          return true;
-        }
-      }
-      // left with only subkeys now
-      if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
-        // skip keys which are for subdirectories of the directory
-        statuses.add(getFileStatus(keyPath));
-      }
-      return true;
-    }
-
-    FileStatus[] getStatuses() {
-      return statuses.toArray(new FileStatus[statuses.size()]);
-    }
-  }
-
-  @Override
-  public FileStatus[] listStatus(Path f) throws IOException {
-    LOG.trace("listStatus() path:{}", f);
-    ListStatusIterator iterator = new ListStatusIterator(f);
-    iterator.iterate();
-    return iterator.getStatuses();
-  }
-
-  @Override
-  public void setWorkingDirectory(Path newDir) {
-    workingDir = newDir;
-  }
-
-  @Override
-  public Path getWorkingDirectory() {
-    return workingDir;
-  }
-
-  /**
-   * Check whether the path is valid and then create directories.
-   * Directory is represented using a key with no value.
-   * All the non-existent parent directories are also created.
-   *
-   * @param path directory path to be created
-   * @return true if directory exists or created successfully.
-   * @throws IOException
-   */
-  private boolean mkdir(Path path) throws IOException {
-    Path fPart = path;
-    Path prevfPart = null;
-    do {
-      LOG.trace("validating path:{}", fPart);
-      try {
-        FileStatus fileStatus = getFileStatus(fPart);
-        if (fileStatus.isDirectory()) {
-          // If path exists and a directory, exit
-          break;
-        } else {
-          // Found a file here, rollback and delete newly created directories
-          LOG.trace("Found a file with same name as directory, path:{}", fPart);
-          if (prevfPart != null) {
-            delete(prevfPart, true);
-          }
-          throw new FileAlreadyExistsException(String.format(
-              "Can't make directory for path '%s', it is a file.", fPart));
-        }
-      } catch (FileNotFoundException fnfe) {
-        LOG.trace("creating directory for fpart:{}", fPart);
-        String key = pathToKey(fPart);
-        String dirKey = addTrailingSlashIfNeeded(key);
-        if (!createDirectory(dirKey)) {
-          // Directory creation failed here,
-          // rollback and delete newly created directories
-          LOG.trace("Directory creation failed, path:{}", fPart);
-          if (prevfPart != null) {
-            delete(prevfPart, true);
-          }
-          return false;
-        }
-      }
-      prevfPart = fPart;
-      fPart = fPart.getParent();
-    } while (fPart != null);
-    return true;
-  }
-
-  @Override
-  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
-    LOG.trace("mkdir() path:{} ", f);
-    String key = pathToKey(f);
-    if (StringUtils.isEmpty(key)) {
-      return false;
-    }
-    return mkdir(f);
-  }
-
-  @Override
-  public FileStatus getFileStatus(Path f) throws IOException {
-    LOG.trace("getFileStatus() path:{}", f);
-    Path qualifiedPath = f.makeQualified(uri, workingDir);
-    String key = pathToKey(qualifiedPath);
-
-    if (key.length() == 0) {
-      return new FileStatus(0, true, 1, 0,
-          bucket.getCreationTime(), qualifiedPath);
-    }
-
-    // consider this a file and get key status
-    OzoneKey meta = getKeyInfo(key);
-    if (meta == null) {
-      key = addTrailingSlashIfNeeded(key);
-      meta = getKeyInfo(key);
-    }
-
-    if (meta == null) {
-      LOG.trace("File:{} not found", f);
-      throw new FileNotFoundException(f + ": No such file or directory!");
-    } else if (isDirectory(meta)) {
-      return new FileStatus(0, true, 1, 0,
-          meta.getModificationTime(), qualifiedPath);
-    } else {
-      //TODO: Fetch replication count from ratis config
-      return new FileStatus(meta.getDataSize(), false, 1,
-            getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
-    }
-  }
-
-  /**
-   * Helper method to fetch the key metadata info.
-   * @param key key whose metadata information needs to be fetched
-   * @return metadata info of the key
-   */
-  private OzoneKey getKeyInfo(String key) {
-    try {
-      return bucket.getKey(key);
-    } catch (IOException e) {
-      LOG.trace("Key:{} does not exists", key);
-      return null;
-    }
-  }
-
-  /**
-   * Helper method to check if an Ozone key is representing a directory.
-   * @param key key to be checked as a directory
-   * @return true if key is a directory, false otherwise
-   */
-  private boolean isDirectory(OzoneKey key) {
-    LOG.trace("key name:{} size:{}", key.getName(),
-        key.getDataSize());
-    return key.getName().endsWith(OZONE_URI_DELIMITER)
-        && (key.getDataSize() == 0);
-  }
-
-  /**
-   * Helper method to create an directory specified by key name in bucket.
-   * @param keyName key name to be created as directory
-   * @return true if the key is created, false otherwise
-   */
-  private boolean createDirectory(String keyName) {
-    try {
-      LOG.trace("creating dir for key:{}", keyName);
-      bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
-      return true;
-    } catch (IOException ioe) {
-      LOG.error("create key failed for key:{}", keyName, ioe);
-      return false;
-    }
-  }
-
-  /**
-   * Helper method to delete an object specified by key name in bucket.
-   * @param keyName key name to be deleted
-   * @return true if the key is deleted, false otherwise
-   */
-  private boolean deleteObject(String keyName) {
-    LOG.trace("issuing delete for key" + keyName);
-    try {
-      bucket.deleteKey(keyName);
-      return true;
-    } catch (IOException ioe) {
-      LOG.error("delete key failed " + ioe.getMessage());
-      return false;
-    }
-  }
-
-  /**
-   * Turn a path (relative or otherwise) into an Ozone key.
-   *
-   * @param path the path of the file.
-   * @return the key of the object that represents the file.
-   */
-  public String pathToKey(Path path) {
-    Objects.requireNonNull(path, "Path can not be null!");
-    if (!path.isAbsolute()) {
-      path = new Path(workingDir, path);
-    }
-    // removing leading '/' char
-    String key = path.toUri().getPath().substring(1);
-    LOG.trace("path for key:{} is:{}", key, path);
-    return key;
-  }
-
-  /**
-   * Add trailing delimiter to path if it is already not present.
-   *
-   * @param key the ozone Key which needs to be appended
-   * @return delimiter appended key
-   */
-  private String addTrailingSlashIfNeeded(String key) {
-    if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
-      return key + OZONE_URI_DELIMITER;
-    } else {
-      return key;
-    }
-  }
-
-  @Override
-  public String toString() {
-    return "OzoneFileSystem{URI=" + uri + ", "
-        + "workingDir=" + workingDir + ", "
-        + "userName=" + userName + ", "
-        + "statistics=" + statistics
-        + "}";
-  }
-
-  private abstract class OzoneListingIterator {
-    private final Path path;
-    private final FileStatus status;
-    private String pathKey;
-    private Iterator<OzoneKey> keyIterator;
-
-    OzoneListingIterator(Path path)
-        throws IOException {
-      this.path = path;
-      this.status = getFileStatus(path);
-      this.pathKey = pathToKey(path);
-      if (status.isDirectory()) {
-        this.pathKey = addTrailingSlashIfNeeded(pathKey);
-      }
-      keyIterator = bucket.listKeys(pathKey);
-    }
-
-    abstract boolean processKey(String key) throws IOException;
-
-    // iterates all the keys in the particular path
-    boolean iterate() throws IOException {
-      LOG.trace("Iterating path {}", path);
-      if (status.isDirectory()) {
-        LOG.trace("Iterating directory:{}", pathKey);
-        while (keyIterator.hasNext()) {
-          OzoneKey key = keyIterator.next();
-          LOG.trace("iterating key:{}", key.getName());
-          if (!processKey(key.getName())) {
-            return false;
-          }
-        }
-        return true;
-      } else {
-        LOG.trace("iterating file:{}", path);
-        return processKey(pathKey);
-      }
-    }
-
-    String getPathKey() {
-      return pathKey;
-    }
-
-    boolean pathIsDirectory() {
-      return status.isDirectory();
-    }
-
-    FileStatus getStatus() {
-      return status;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
deleted file mode 100644
index 93e82c3..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Ozone Filesystem.
- *
- * Except for the exceptions, it should all be hidden as implementation details.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
deleted file mode 100644
index ad21f28..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Test OzoneFSInputStream by reading through multiple interfaces.
- */
-public class TestOzoneFSInputStream {
-  private static MiniOzoneCluster cluster = null;
-  private static FileSystem fs;
-  private static StorageHandler storageHandler;
-  private static Path filePath = null;
-  private static byte[] data = null;
-
-  /**
-   * Create a MiniDFSCluster for testing.
-   * <p>
-   * Ozone is made active by setting OZONE_ENABLED = true and
-   * OZONE_HANDLER_TYPE_KEY = "distributed"
-   *
-   * @throws IOException
-   */
-  @BeforeClass
-  public static void init() throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
-    cluster = MiniOzoneCluster.newBuilder(conf)
-        .setNumDatanodes(10)
-        .build();
-    cluster.waitForClusterToBeReady();
-    storageHandler =
-        new ObjectStoreHandler(conf).getStorageHandler();
-
-    // create a volume and a bucket to be used by OzoneFileSystem
-    String userName = "user" + RandomStringUtils.randomNumeric(5);
-    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
-        null, null, null, null);
-    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
-    volumeArgs.setUserName(userName);
-    volumeArgs.setAdminName(adminName);
-    storageHandler.createVolume(volumeArgs);
-    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
-    storageHandler.createBucket(bucketArgs);
-
-    // Fetch the host and port for File System init
-    DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
-        .getDatanodeDetails();
-    int port = datanodeDetails
-        .getPort(DatanodeDetails.Port.Name.REST).getValue();
-    String host = datanodeDetails.getHostName();
-
-    // Set the fs.defaultFS and start the filesystem
-    String uri = String.format("%s://%s.%s/",
-        OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
-    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
-    fs =  FileSystem.get(conf);
-    int fileLen = 100 * 1024 * 1024;
-    data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
-    filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
-    try (FSDataOutputStream stream = fs.create(filePath)) {
-      stream.write(data);
-    }
-  }
-
-  /**
-   * Shutdown MiniDFSCluster.
-   */
-  @AfterClass
-  public static void shutdown() throws IOException {
-    fs.close();
-    storageHandler.close();
-    cluster.shutdown();
-  }
-
-  @Test
-  public void testO3FSSingleByteRead() throws IOException {
-    FSDataInputStream inputStream = fs.open(filePath);
-    byte[] value = new byte[data.length];
-    int i = 0;
-    while(true) {
-      int val = inputStream.read();
-      if (val == -1) {
-        break;
-      }
-      value[i] = (byte)val;
-      Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
-      i++;
-    }
-    Assert.assertEquals(i, data.length);
-    Assert.assertTrue(Arrays.equals(value, data));
-    inputStream.close();
-  }
-
-  @Test
-  public void testO3FSMultiByteRead() throws IOException {
-    FSDataInputStream inputStream = fs.open(filePath);
-    byte[] value = new byte[data.length];
-    byte[] tmp = new byte[1* 1024 *1024];
-    int i = 0;
-    while(true) {
-      int val = inputStream.read(tmp);
-      if (val == -1) {
-        break;
-      }
-      System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
-      i++;
-    }
-    Assert.assertEquals(i * tmp.length, data.length);
-    Assert.assertTrue(Arrays.equals(value, data));
-    inputStream.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
deleted file mode 100644
index a225702..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.junit.After;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test OzoneFileSystem Interfaces.
- *
- * This test will test the various interfaces i.e.
- * create, read, write, getFileStatus
- */
-@RunWith(Parameterized.class)
-public class TestOzoneFileInterfaces {
-
-  private String rootPath;
-  private String userName;
-
-  /**
-   * Parameter class to set absolute url/defaultFS handling.
-   * <p>
-   * Hadoop file systems could be used in multiple ways: Using the defaultfs
-   * and file path without the schema, or use absolute url-s even with
-   * different defaultFS. This parameter matrix would test both the use cases.
-   */
-  @Parameters
-  public static Collection<Object[]> data() {
-    return Arrays.asList(new Object[][] {{false, true}, {true, false}});
-  }
-
-  private boolean setDefaultFs;
-
-  private boolean useAbsolutePath;
-
-  private static MiniOzoneCluster cluster = null;
-
-  private static FileSystem fs;
-
-  private static StorageHandler storageHandler;
-
-  public TestOzoneFileInterfaces(boolean setDefaultFs,
-      boolean useAbsolutePath) {
-    this.setDefaultFs = setDefaultFs;
-    this.useAbsolutePath = useAbsolutePath;
-  }
-
-  @Before
-  public void init() throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
-    cluster = MiniOzoneCluster.newBuilder(conf)
-        .setNumDatanodes(3)
-        .build();
-    cluster.waitForClusterToBeReady();
-    storageHandler =
-        new ObjectStoreHandler(conf).getStorageHandler();
-
-    // create a volume and a bucket to be used by OzoneFileSystem
-    userName = "user" + RandomStringUtils.randomNumeric(5);
-    String adminName = "admin" + RandomStringUtils.randomNumeric(5);
-    String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
-    String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-    UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
-        null, null, null, null);
-    VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
-    volumeArgs.setUserName(userName);
-    volumeArgs.setAdminName(adminName);
-    storageHandler.createVolume(volumeArgs);
-    BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
-    storageHandler.createBucket(bucketArgs);
-
-    rootPath = String
-        .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
-            volumeName);
-    if (setDefaultFs) {
-      // Set the fs.defaultFS and start the filesystem
-      conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
-      fs = FileSystem.get(conf);
-    } else {
-      fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
-    }
-  }
-
-  @After
-  public void teardown() throws IOException {
-    if (cluster != null) {
-      cluster.shutdown();
-    }
-    IOUtils.closeQuietly(fs);
-    IOUtils.closeQuietly(storageHandler);
-  }
-
-  @Test
-  public void testFileSystemInit() throws IOException {
-    if (setDefaultFs) {
-      assertTrue(
-          "The initialized file system is not OzoneFileSystem but " +
-              fs.getClass(),
-          fs instanceof OzoneFileSystem);
-      assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
-    }
-  }
-
-  @Test
-  public void testOzFsReadWrite() throws IOException {
-    long currentTime = Time.now();
-    int stringLen = 20;
-    String data = RandomStringUtils.randomAlphanumeric(stringLen);
-    String filePath = RandomStringUtils.randomAlphanumeric(5);
-    Path path = createPath("/" + filePath);
-    try (FSDataOutputStream stream = fs.create(path)) {
-      stream.writeBytes(data);
-    }
-
-    FileStatus status = fs.getFileStatus(path);
-    // The timestamp of the newly created file should always be greater than
-    // the time when the test was started
-    assertTrue("Modification time has not been recorded: " + status,
-        status.getModificationTime() > currentTime);
-
-    try (FSDataInputStream inputStream = fs.open(path)) {
-      byte[] buffer = new byte[stringLen];
-      inputStream.readFully(0, buffer);
-      String out = new String(buffer, 0, buffer.length);
-      assertEquals(data, out);
-    }
-  }
-
-
-  @Test
-  public void testDirectory() throws IOException {
-    String dirPath = RandomStringUtils.randomAlphanumeric(5);
-    Path path = createPath("/" + dirPath);
-    assertTrue("Makedirs returned with false for the path " + path,
-        fs.mkdirs(path));
-
-    FileStatus status = fs.getFileStatus(path);
-    assertTrue("The created path is not directory.", status.isDirectory());
-
-    assertEquals(0, status.getLen());
-
-    FileStatus[] statusList = fs.listStatus(createPath("/"));
-    assertEquals(1, statusList.length);
-    assertEquals(status, statusList[0]);
-
-    FileStatus statusRoot = fs.getFileStatus(createPath("/"));
-    assertTrue("Root dir (/) is not a directory.", status.isDirectory());
-    assertEquals(0, status.getLen());
-
-
-  }
-
-  @Test
-  public void testPathToKey() throws Exception {
-    OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
-
-    assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
-
-    assertEquals("user/" + getCurrentUser() + "/key1/key2",
-        ozoneFs.pathToKey(new Path("key1/key2")));
-
-    assertEquals("key1/key2",
-        ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
-  }
-
-  private String getCurrentUser() {
-    try {
-      return UserGroupInformation.getCurrentUser().getShortUserName();
-    } catch (IOException e) {
-      return OZONE_DEFAULT_USER;
-    }
-  }
-
-  private Path createPath(String relativePath) {
-    if (useAbsolutePath) {
-      return new Path(
-          rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
-    } else {
-      return new Path(relativePath);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
deleted file mode 100644
index dd54315..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests creating files.
- */
-public class ITestOzoneContractCreate extends AbstractContractCreateTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
deleted file mode 100644
index f0a3d8d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering deletes.
- */
-public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
deleted file mode 100644
index 134a9ad..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-
-/**
- * Contract test suite covering S3A integration with DistCp.
- * Uses the block output stream, buffered to disk. This is the
- * recommended output mechanism for DistCP due to its scalability.
- */
-public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected OzoneContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
deleted file mode 100644
index 98bbb14..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering getFileStatus.
- */
-public class ITestOzoneContractGetFileStatus
-    extends AbstractContractGetFileStatusTest {
-
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-
-  @Override
-  public void teardown() throws Exception {
-    getLog().info("FS details {}", getFileSystem());
-    super.teardown();
-  }
-
-  @Override
-  protected Configuration createConfiguration() {
-    return super.createConfiguration();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
deleted file mode 100644
index bc0de5d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Test dir operations on Ozone.
- */
-public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
deleted file mode 100644
index 0bc57d4..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests opening files.
- */
-public class ITestOzoneContractOpen extends AbstractContractOpenTest {
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
deleted file mode 100644
index 8ce1d1b..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering rename.
- */
-public class ITestOzoneContractRename extends AbstractContractRenameTest {
-
-  @BeforeClass
-  public static void createCluster() throws IOException {
-    OzoneContract.createCluster();
-  }
-
-  @AfterClass
-  public static void teardownCluster() throws IOException {
-    OzoneContract.destroyCluster();
-  }
-
-  @Override
-  protected AbstractFSContract createContract(Configuration conf) {
-    return new OzoneContract(conf);
-  }
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[35/50] [abbrv] hadoop git commit: HADOOP-15495. Upgrade commons-lang version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by Takanobu Asanuma.

Posted by gi...@apache.org.
HADOOP-15495. Upgrade commons-lang version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2399d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2399d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2399d6

Branch: refs/heads/HADOOP-15461
Commit: 2b2399d623539ab68e71a38fa9fbfc9a405bddb8
Parents: 8752a48
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jun 28 14:29:40 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Jun 28 14:37:22 2018 +0900

----------------------------------------------------------------------
 .../hadoop-client-minicluster/pom.xml           |  8 ---
 hadoop-common-project/hadoop-common/pom.xml     |  5 --
 .../hadoop/conf/ReconfigurationServlet.java     | 42 ++++++-------
 .../apache/hadoop/crypto/key/KeyProvider.java   |  4 +-
 .../org/apache/hadoop/crypto/key/KeyShell.java  |  2 +-
 .../main/java/org/apache/hadoop/fs/FsShell.java |  3 +-
 .../main/java/org/apache/hadoop/fs/Path.java    |  2 +-
 .../java/org/apache/hadoop/fs/shell/Count.java  |  2 +-
 .../apache/hadoop/io/ElasticByteBufferPool.java |  2 +-
 .../apache/hadoop/io/erasurecode/ECSchema.java  |  4 +-
 .../nativeio/SharedFileDescriptorFactory.java   |  2 +-
 .../org/apache/hadoop/ipc/CallerContext.java    |  4 +-
 .../apache/hadoop/ipc/DecayRpcScheduler.java    |  4 +-
 .../org/apache/hadoop/ipc/FairCallQueue.java    |  4 +-
 .../hadoop/metrics2/MetricsJsonBuilder.java     |  2 +-
 .../hadoop/metrics2/lib/MethodMetric.java       |  2 +-
 .../metrics2/lib/MutableMetricsFactory.java     |  2 +-
 .../hadoop/metrics2/lib/MutableQuantiles.java   |  2 +-
 .../metrics2/lib/MutableRollingAverages.java    |  2 +-
 .../apache/hadoop/metrics2/lib/MutableStat.java |  2 +-
 .../metrics2/sink/RollingFileSystemSink.java    |  2 +-
 .../org/apache/hadoop/net/TableMapping.java     |  2 +-
 .../apache/hadoop/net/unix/DomainSocket.java    |  2 +-
 .../hadoop/net/unix/DomainSocketWatcher.java    |  2 +-
 .../security/ShellBasedUnixGroupsMapping.java   |  2 +-
 .../hadoop/security/alias/CredentialShell.java  |  2 +-
 .../hadoop/security/http/CrossOriginFilter.java |  2 +-
 .../hadoop/security/token/DtFileOperations.java |  2 +-
 .../org/apache/hadoop/tools/TableListing.java   |  6 +-
 .../org/apache/hadoop/util/StringUtils.java     | 64 +++++++++++++++++++-
 .../apache/hadoop/conf/TestConfiguration.java   |  2 +-
 .../conf/TestConfigurationFieldsBase.java       |  2 +-
 .../crypto/random/TestOsSecureRandom.java       |  2 +-
 .../org/apache/hadoop/fs/FSTestWrapper.java     |  2 +-
 .../apache/hadoop/fs/FileContextTestHelper.java |  1 -
 .../hadoop/fs/TestDFCachingGetSpaceUsed.java    |  2 +-
 .../fs/TestFileSystemStorageStatistics.java     | 18 +++---
 .../hadoop/fs/shell/TestCopyFromLocal.java      | 10 +--
 .../TestSharedFileDescriptorFactory.java        |  2 +-
 .../org/apache/hadoop/ipc/TestProtoBufRpc.java  |  2 +-
 .../java/org/apache/hadoop/ipc/TestSaslRPC.java |  2 +-
 .../org/apache/hadoop/net/TestNetUtils.java     |  2 +-
 .../hadoop/net/unix/TestDomainSocket.java       |  2 +-
 ...tionTokenAuthenticationHandlerWithMocks.java |  2 +-
 .../AbstractServiceLauncherTestBase.java        |  2 +-
 .../apache/hadoop/test/GenericTestUtils.java    |  4 +-
 .../hadoop/util/TestShutdownHookManager.java    |  2 +-
 .../apache/hadoop/util/TestSignalLogger.java    |  2 +-
 .../hdfs/server/namenode/FSNamesystemLock.java  |  2 +-
 hadoop-project/pom.xml                          |  5 --
 .../aliyun/oss/AliyunCredentialsProvider.java   |  2 +-
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      |  2 +-
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java |  2 +-
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java    |  2 +-
 .../fs/aliyun/oss/AliyunOSSTestUtils.java       |  2 +-
 .../fs/s3a/AWSCredentialProviderList.java       |  2 +-
 .../fs/s3a/BasicAWSCredentialsProvider.java     |  2 +-
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  4 +-
 .../apache/hadoop/fs/s3a/S3AInputStream.java    |  2 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  2 +-
 .../fs/s3a/SimpleAWSCredentialsProvider.java    |  2 +-
 .../fs/s3a/TemporaryAWSCredentialsProvider.java |  2 +-
 .../s3a/auth/AssumedRoleCredentialProvider.java |  2 +-
 .../s3a/commit/files/SinglePendingCommit.java   |  2 +-
 .../hadoop/fs/s3a/commit/files/SuccessData.java |  2 +-
 .../fs/s3a/commit/magic/MagicCommitTracker.java |  2 +-
 .../hadoop/fs/s3a/commit/staging/Paths.java     |  2 +-
 .../fs/s3a/s3guard/DynamoDBClientFactory.java   |  2 +-
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   |  2 +-
 .../fs/s3a/s3guard/LocalMetadataStore.java      |  2 +-
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java      |  2 +-
 .../hadoop/fs/s3native/S3xLoginHelper.java      |  4 +-
 .../hadoop/fs/s3a/ITestS3AConfiguration.java    |  4 +-
 .../ITestS3AEncryptionSSEKMSUserDefinedKey.java |  2 +-
 ...onSSEKMSUserDefinedKeyBlockOutputStream.java |  2 +-
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |  2 +-
 .../fs/s3a/commit/AbstractCommitITest.java      |  2 +-
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java |  2 +-
 .../hadoop/fs/adl/TestAzureADTokenProvider.java |  2 +-
 .../fs/azure/AzureNativeFileSystemStore.java    |  2 +-
 .../hadoop/fs/azure/BlockBlobAppendStream.java  |  2 +-
 .../fs/azure/ClientThrottlingAnalyzer.java      |  2 +-
 .../hadoop/fs/azure/NativeAzureFileSystem.java  |  2 +-
 .../hadoop/fs/azure/PageBlobOutputStream.java   |  2 +-
 .../fs/azure/RemoteWasbAuthorizerImpl.java      |  2 +-
 .../fs/azure/SecureWasbRemoteCallHelper.java    |  2 +-
 .../fs/azure/ITestBlobDataValidation.java       |  2 +-
 .../azure/ITestNativeAzureFileSystemAppend.java |  2 +-
 .../hadoop/fs/azure/MockStorageInterface.java   | 12 ++--
 ...estNativeAzureFileSystemBlockCompaction.java |  2 +-
 .../fs/azure/integration/AzureTestUtils.java    |  2 +-
 .../org/apache/hadoop/tools/DistCpOptions.java  |  2 +-
 .../org/apache/hadoop/tools/OptionsParser.java  |  2 +-
 .../apache/hadoop/tools/mapred/CopyMapper.java  |  2 +-
 .../mapred/gridmix/ClusterSummarizer.java       |  2 +-
 .../mapred/gridmix/ExecutionSummarizer.java     |  4 +-
 .../hadoop/mapred/gridmix/JobFactory.java       |  2 +-
 .../mapred/gridmix/RandomTextDataGenerator.java |  2 +-
 .../translator/impl/BaseLogParser.java          |  1 -
 .../WordListAnonymizerUtility.java              |  2 +-
 .../util/MapReduceJobPropertiesParser.java      |  2 +-
 .../hadoop/streaming/TestUnconsumedInput.java   |  2 +-
 102 files changed, 207 insertions(+), 168 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index b9363de..6fa24b4 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -167,10 +167,6 @@
           <artifactId>commons-io</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>commons-lang</groupId>
-          <artifactId>commons-lang</artifactId>
-        </exclusion>
-        <exclusion>
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>
         </exclusion>
@@ -496,10 +492,6 @@
           <artifactId>commons-codec</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>commons-lang</groupId>
-          <artifactId>commons-lang</artifactId>
-        </exclusion>
-        <exclusion>
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>
         </exclusion>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 1a16dc4..67a5a54 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -157,11 +157,6 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>commons-lang</groupId>
-      <artifactId>commons-lang</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
       <groupId>commons-beanutils</groupId>
       <artifactId>commons-beanutils</artifactId>
       <scope>compile</scope>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
index 5a616f7..c5bdf4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.conf;
 
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
 
 import java.util.Collection;
 import java.util.Enumeration;
@@ -72,10 +72,10 @@ public class ReconfigurationServlet extends HttpServlet {
   private void printHeader(PrintWriter out, String nodeName) {
     out.print("<html><head>");
     out.printf("<title>%s Reconfiguration Utility</title>%n",
-               StringEscapeUtils.escapeHtml(nodeName));
+               StringEscapeUtils.escapeHtml4(nodeName));
     out.print("</head><body>\n");
     out.printf("<h1>%s Reconfiguration Utility</h1>%n",
-               StringEscapeUtils.escapeHtml(nodeName));
+               StringEscapeUtils.escapeHtml4(nodeName));
   }
 
   private void printFooter(PrintWriter out) {
@@ -103,20 +103,20 @@ public class ReconfigurationServlet extends HttpServlet {
       out.print("<tr><td>");
       if (!reconf.isPropertyReconfigurable(c.prop)) {
         out.print("<font color=\"red\">" + 
-                  StringEscapeUtils.escapeHtml(c.prop) + "</font>");
+                  StringEscapeUtils.escapeHtml4(c.prop) + "</font>");
         changeOK = false;
       } else {
-        out.print(StringEscapeUtils.escapeHtml(c.prop));
+        out.print(StringEscapeUtils.escapeHtml4(c.prop));
         out.print("<input type=\"hidden\" name=\"" +
-                  StringEscapeUtils.escapeHtml(c.prop) + "\" value=\"" +
-                  StringEscapeUtils.escapeHtml(c.newVal) + "\"/>");
+                  StringEscapeUtils.escapeHtml4(c.prop) + "\" value=\"" +
+                  StringEscapeUtils.escapeHtml4(c.newVal) + "\"/>");
       }
       out.print("</td><td>" +
                 (c.oldVal == null ? "<it>default</it>" : 
-                 StringEscapeUtils.escapeHtml(c.oldVal)) +
+                 StringEscapeUtils.escapeHtml4(c.oldVal)) +
                 "</td><td>" +
                 (c.newVal == null ? "<it>default</it>" : 
-                 StringEscapeUtils.escapeHtml(c.newVal)) +
+                 StringEscapeUtils.escapeHtml4(c.newVal)) +
                 "</td>");
       out.print("</tr>\n");
     }
@@ -147,9 +147,9 @@ public class ReconfigurationServlet extends HttpServlet {
     synchronized(oldConf) {
       while (params.hasMoreElements()) {
         String rawParam = params.nextElement();
-        String param = StringEscapeUtils.unescapeHtml(rawParam);
+        String param = StringEscapeUtils.unescapeHtml4(rawParam);
         String value =
-          StringEscapeUtils.unescapeHtml(req.getParameter(rawParam));
+          StringEscapeUtils.unescapeHtml4(req.getParameter(rawParam));
         if (value != null) {
           if (value.equals(newConf.getRaw(param)) || value.equals("default") ||
               value.equals("null") || value.isEmpty()) {
@@ -157,8 +157,8 @@ public class ReconfigurationServlet extends HttpServlet {
                  value.isEmpty()) && 
                 oldConf.getRaw(param) != null) {
               out.println("<p>Changed \"" + 
-                          StringEscapeUtils.escapeHtml(param) + "\" from \"" +
-                          StringEscapeUtils.escapeHtml(oldConf.getRaw(param)) +
+                          StringEscapeUtils.escapeHtml4(param) + "\" from \"" +
+                          StringEscapeUtils.escapeHtml4(oldConf.getRaw(param)) +
                           "\" to default</p>");
               reconf.reconfigureProperty(param, null);
             } else if (!value.equals("default") && !value.equals("null") &&
@@ -168,16 +168,16 @@ public class ReconfigurationServlet extends HttpServlet {
               // change from default or value to different value
               if (oldConf.getRaw(param) == null) {
                 out.println("<p>Changed \"" + 
-                            StringEscapeUtils.escapeHtml(param) + 
+                            StringEscapeUtils.escapeHtml4(param) +
                             "\" from default to \"" +
-                            StringEscapeUtils.escapeHtml(value) + "\"</p>");
+                            StringEscapeUtils.escapeHtml4(value) + "\"</p>");
               } else {
                 out.println("<p>Changed \"" + 
-                            StringEscapeUtils.escapeHtml(param) + "\" from \"" +
-                            StringEscapeUtils.escapeHtml(oldConf.
+                            StringEscapeUtils.escapeHtml4(param) + "\" from \"" +
+                            StringEscapeUtils.escapeHtml4(oldConf.
                                                          getRaw(param)) +
                             "\" to \"" +
-                            StringEscapeUtils.escapeHtml(value) + "\"</p>");
+                            StringEscapeUtils.escapeHtml4(value) + "\"</p>");
               }
               reconf.reconfigureProperty(param, value);
             } else {
@@ -185,10 +185,10 @@ public class ReconfigurationServlet extends HttpServlet {
             }
           } else {
             // parameter value != newConf value
-            out.println("<p>\"" + StringEscapeUtils.escapeHtml(param) + 
+            out.println("<p>\"" + StringEscapeUtils.escapeHtml4(param) +
                         "\" not changed because value has changed from \"" +
-                        StringEscapeUtils.escapeHtml(value) + "\" to \"" +
-                        StringEscapeUtils.escapeHtml(newConf.getRaw(param)) +
+                        StringEscapeUtils.escapeHtml4(value) + "\" to \"" +
+                        StringEscapeUtils.escapeHtml4(newConf.getRaw(param)) +
                         "\" since approval</p>");
           }
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 050540b..286312c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -33,8 +33,8 @@ import java.util.Map;
 
 import com.google.gson.stream.JsonReader;
 import com.google.gson.stream.JsonWriter;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index 9fdf242..fa84c47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -27,7 +27,7 @@ import java.util.Map;
 
 import com.google.common.annotations.VisibleForTesting;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider.Metadata;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 94d3389..5be6e5f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
 
-import org.apache.commons.lang.WordUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
@@ -275,7 +274,7 @@ public class FsShell extends Configured implements Tool {
         listing = null;
       }
 
-      for (String descLine : WordUtils.wrap(
+      for (String descLine : StringUtils.wrap(
           line, MAX_LINE_WIDTH, "\n", true).split("\n")) {
         out.println(prefix + descLine);
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
index 252b3cc..b6244d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
@@ -27,7 +27,7 @@ import java.net.URISyntaxException;
 import java.util.regex.Pattern;
 
 import org.apache.avro.reflect.Stringable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 8f6fc4d..011e489 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
index 9dd7771..bbedf2a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
@@ -18,7 +18,7 @@
 package org.apache.hadoop.io;
 
 import com.google.common.collect.ComparisonChain;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 
 import java.nio.ByteBuffer;
 import java.util.Map;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
index f008e24..0f95058 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
@@ -22,8 +22,8 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
index 4126344..4d820c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
@@ -21,7 +21,7 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.FileDescriptor;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.slf4j.Logger;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index bdfa471..b156d1f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.ipc;
 
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index 0a00ca7..f12ecb6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -39,7 +39,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.AtomicDoubleArray;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.metrics2.MetricsCollector;
@@ -429,7 +429,7 @@ public class DecayRpcScheduler implements RpcScheduler,
       updateAverageResponseTime(true);
     } catch (Exception ex) {
       LOG.error("decayCurrentCounts exception: " +
-          ExceptionUtils.getFullStackTrace(ex));
+          ExceptionUtils.getStackTrace(ex));
       throw ex;
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 6d9ea3e..3a8c83d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
 import org.apache.hadoop.metrics2.util.MBeans;
@@ -286,7 +286,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
    */
   @Override
   public Iterator<E> iterator() {
-    throw new NotImplementedException();
+    throw new NotImplementedException("Code is not implemented");
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
index ce6fbe1..1d62c0a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.metrics2;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.codehaus.jackson.map.ObjectMapper;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
index 3d7a90e..9b54adc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.metrics2.lib;
 import java.lang.reflect.Method;
 
 import static com.google.common.base.Preconditions.*;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.metrics2.MetricsException;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
index b2042e7..a3ca98d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.metrics2.lib;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.metrics2.MetricsException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
index cc32975..6b30618 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.metrics2.MetricsInfo;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
index 053cb55..22c288a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.metrics2.MetricsInfo;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
index 92fe3d1..5ef3178 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.metrics2.lib;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.metrics2.MetricsInfo;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 0f6e9a9..92ac952 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -37,7 +37,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
index ead9a74..45759df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
index ac118c0..9693220 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
@@ -28,7 +28,7 @@ import java.nio.channels.ClosedChannelException;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.ByteBuffer;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.apache.hadoop.util.CloseableReferenceCount;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
index c7af97f..e36399f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
@@ -32,7 +32,7 @@ import java.util.Map;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.util.NativeCodeLoader;
 
 import com.google.common.annotations.VisibleForTesting;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index 976ddba..31f4398 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index 6085121..0a00d79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -27,7 +27,7 @@ import java.util.List;
 
 import com.google.common.annotations.VisibleForTesting;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.tools.CommandShell;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
index 34d9fe2..02c168f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
@@ -34,7 +34,7 @@ import javax.servlet.ServletResponse;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.util.stream.Collectors;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index d36ad9b..f154f2d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -26,7 +26,7 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.ServiceLoader;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
index bc2e2d4..85015fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.tools;
 import java.util.ArrayList;
 import java.util.LinkedList;
 
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.WordUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 
 /**
@@ -103,7 +102,8 @@ public class TableListing {
       // Line-wrap if it's too long
       String[] lines = new String[] {raw};
       if (wrap) {
-        lines = WordUtils.wrap(lines[0], wrapWidth, "\n", true).split("\n");
+        lines = org.apache.hadoop.util.StringUtils.wrap(lines[0], wrapWidth,
+            "\n", true).split("\n");
       }
       for (int i=0; i<lines.length; i++) {
         if (justification == Justification.LEFT) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 33a2010..3db805f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -35,7 +35,7 @@ import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -987,7 +987,7 @@ public class StringUtils {
     String[] words = split(StringUtils.toLowerCase(s), ESCAPE_CHAR,  '_');
 
     for (String word : words)
-      sb.append(org.apache.commons.lang.StringUtils.capitalize(word));
+      sb.append(org.apache.commons.lang3.StringUtils.capitalize(word));
 
     return sb.toString();
   }
@@ -1183,4 +1183,64 @@ public class StringUtils {
     return true;
   }
 
+  /**
+   * Same as WordUtils#wrap in commons-lang 2.6. Unlike commons-lang3, leading
+   * spaces on the first line are NOT stripped.
+   *
+   * @param str  the String to be word wrapped, may be null
+   * @param wrapLength  the column to wrap the words at, less than 1 is treated
+   *                   as 1
+   * @param newLineStr  the string to insert for a new line,
+   *  <code>null</code> uses the system property line separator
+   * @param wrapLongWords  true if long words (such as URLs) should be wrapped
+   * @return a line with newlines inserted, <code>null</code> if null input
+   */
+  public static String wrap(String str, int wrapLength, String newLineStr,
+      boolean wrapLongWords) {
+    if(str == null) {
+      return null;
+    } else {
+      if(newLineStr == null) {
+        newLineStr = System.lineSeparator();
+      }
+
+      if(wrapLength < 1) {
+        wrapLength = 1;
+      }
+
+      int inputLineLength = str.length();
+      int offset = 0;
+      StringBuffer wrappedLine = new StringBuffer(inputLineLength + 32);
+
+      while(inputLineLength - offset > wrapLength) {
+        if(str.charAt(offset) == 32) {
+          ++offset;
+        } else {
+          int spaceToWrapAt = str.lastIndexOf(32, wrapLength + offset);
+          if(spaceToWrapAt >= offset) {
+            wrappedLine.append(str.substring(offset, spaceToWrapAt));
+            wrappedLine.append(newLineStr);
+            offset = spaceToWrapAt + 1;
+          } else if(wrapLongWords) {
+            wrappedLine.append(str.substring(offset, wrapLength + offset));
+            wrappedLine.append(newLineStr);
+            offset += wrapLength;
+          } else {
+            spaceToWrapAt = str.indexOf(32, wrapLength + offset);
+            if(spaceToWrapAt >= 0) {
+              wrappedLine.append(str.substring(offset, spaceToWrapAt));
+              wrappedLine.append(newLineStr);
+              offset = spaceToWrapAt + 1;
+            } else {
+              wrappedLine.append(str.substring(offset));
+              offset = inputLineLength;
+            }
+          }
+        }
+      }
+
+      wrappedLine.append(str.substring(offset));
+      return wrappedLine.toString();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index e865bf1..2361626 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.conf.StorageUnit.TB;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.*;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration.IntegerRanges;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 7f27d7d..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.conf;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
index 6c2e5b8..2ea4523 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.crypto.random;
 import java.io.IOException;
 import java.util.Arrays;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.junit.Assume;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
index da07105..8cbca8e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.fs;
 
 import java.io.IOException;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.test.GenericTestUtils;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
index 1f37f74..b5307a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.io.FileNotFoundException;
 import java.util.EnumSet;
 
-import org.apache.commons.lang.RandomStringUtils;
 import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.fs.Options.CreateOpts.BlockSize;
 import org.apache.hadoop.io.IOUtils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
index 3def5d5..6b9a34c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.fs;
 
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Before;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
index 597eb93..fa68264 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs;
 
-import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.fs.StorageStatistics.LongStatistic;
 
 import org.junit.Before;
@@ -67,15 +67,15 @@ public class TestFileSystemStorageStatistics {
 
   @Before
   public void setup() {
-    statistics.incrementBytesRead(RandomUtils.nextInt(100));
-    statistics.incrementBytesWritten(RandomUtils.nextInt(100));
-    statistics.incrementLargeReadOps(RandomUtils.nextInt(100));
-    statistics.incrementWriteOps(RandomUtils.nextInt(100));
+    statistics.incrementBytesRead(RandomUtils.nextInt(0, 100));
+    statistics.incrementBytesWritten(RandomUtils.nextInt(0, 100));
+    statistics.incrementLargeReadOps(RandomUtils.nextInt(0, 100));
+    statistics.incrementWriteOps(RandomUtils.nextInt(0, 100));
 
-    statistics.incrementBytesReadByDistance(0, RandomUtils.nextInt(100));
-    statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(100));
-    statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(100));
-    statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(100));
+    statistics.incrementBytesReadByDistance(0, RandomUtils.nextInt(0, 100));
+    statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(0, 100));
+    statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(0, 100));
+    statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(0, 100));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
index 8e60540..e7f36fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.fs.shell;
 
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileSystem;
@@ -56,11 +56,11 @@ public class TestCopyFromLocal {
     fs.mkdirs(toDirPath);
 
     int numTotalFiles = 0;
-    int numDirs = RandomUtils.nextInt(5);
+    int numDirs = RandomUtils.nextInt(0, 5);
     for (int dirCount = 0; dirCount < numDirs; ++dirCount) {
       Path subDirPath = new Path(fromDirPath, "subdir" + dirCount);
       fs.mkdirs(subDirPath);
-      int numFiles = RandomUtils.nextInt(10);
+      int numFiles = RandomUtils.nextInt(0, 10);
       for (int fileCount = 0; fileCount < numFiles; ++fileCount) {
         numTotalFiles++;
         Path subFile = new Path(subDirPath, "file" + fileCount);
@@ -115,7 +115,7 @@ public class TestCopyFromLocal {
     Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
     int numFiles = TestCopyFromLocal.initialize(dir);
     int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
-    int randThreads = RandomUtils.nextInt(maxThreads - 1) + 1;
+    int randThreads = RandomUtils.nextInt(0, maxThreads - 1) + 1;
     String numThreads = Integer.toString(randThreads);
     run(new TestMultiThreadedCopy(randThreads,
         randThreads == 1 ? 0 : numFiles), "-t", numThreads,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
index fbe3fb8..17be587 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
@@ -26,7 +26,7 @@ import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.test.GenericTestUtils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
index 41ae910..5fbd957 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.ipc;
 import com.google.protobuf.BlockingService;
 import com.google.protobuf.RpcController;
 import com.google.protobuf.ServiceException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.ipc.metrics.RpcMetrics;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 0b463a5..5200420 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.ipc;
 
 import com.google.protobuf.ServiceException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index 30176f2..62bd1b1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -38,7 +38,7 @@ import java.util.Enumeration;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.KerberosAuthException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
index 28b3cbe..c0d204f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
@@ -43,7 +43,7 @@ import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.unix.DomainSocket.DomainChannel;
 import org.apache.hadoop.test.GenericTestUtils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
index 9357f48..0f8f1e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
@@ -21,7 +21,7 @@ import static org.apache.hadoop.security.token.delegation.web.DelegationTokenAut
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.authentication.client.AuthenticationException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
index 127b0b3..d7c8631 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.service.launcher;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.service.ServiceOperations;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 53eb2be..3e9da1b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.fs.FileUtil;
@@ -344,7 +344,7 @@ public abstract class GenericTestUtils {
       throw new AssertionError(E_NULL_THROWABLE_STRING, t);
     }
     if (expectedText != null && !msg.contains(expectedText)) {
-      String prefix = org.apache.commons.lang.StringUtils.isEmpty(message)
+      String prefix = org.apache.commons.lang3.StringUtils.isEmpty(message)
           ? "" : (message + ": ");
       throw new AssertionError(
           String.format("%s Expected to find '%s' %s: %s",

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
index 2aa5e95..d539823 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.util;
 
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.slf4j.LoggerFactory;
 import org.junit.Assert;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
index a9fa4c6..b61cebc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.util;
 
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Test;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index f8e69e2..5992e54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -332,7 +332,7 @@ class FSNamesystemLock {
 
   private static String getMetricName(String operationName, boolean isWrite) {
     return (isWrite ? WRITE_LOCK_METRIC_PREFIX : READ_LOCK_METRIC_PREFIX) +
-        org.apache.commons.lang.StringUtils.capitalize(operationName) +
+        org.apache.commons.lang3.StringUtils.capitalize(operationName) +
         LOCK_METRIC_SUFFIX;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dfd1eac..8e28afe 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1044,11 +1044,6 @@
         <version>4.11</version>
       </dependency>
       <dependency>
-        <groupId>commons-lang</groupId>
-        <artifactId>commons-lang</artifactId>
-        <version>2.6</version>
-      </dependency>
-      <dependency>
         <groupId>commons-collections</groupId>
         <artifactId>commons-collections</artifactId>
         <version>3.2.2</version>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
index 58c14a9..32367af 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
@@ -22,7 +22,7 @@ import com.aliyun.oss.common.auth.Credentials;
 import com.aliyun.oss.common.auth.CredentialsProvider;
 import com.aliyun.oss.common.auth.DefaultCredentials;
 import com.aliyun.oss.common.auth.InvalidCredentialsException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 
 import java.io.IOException;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 93e31d5..4fbb6fb 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.CreateFlag;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index cc050c8..5e21759 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -45,7 +45,7 @@ import com.aliyun.oss.model.UploadPartCopyResult;
 import com.aliyun.oss.model.UploadPartRequest;
 import com.aliyun.oss.model.UploadPartResult;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FileStatus;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index 2fe06c1..a7536d6 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -23,7 +23,7 @@ import java.io.IOException;
 
 import com.aliyun.oss.common.auth.CredentialsProvider;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.security.ProviderUtils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
index 901cb2b..79e0de3 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.aliyun.oss;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.junit.internal.AssumptionViolatedException;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
index e0bee0f..10201f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -23,7 +23,7 @@ import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.IOUtils;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
index b1899e2..01bcc6a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.AWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4b0c208..737d7da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -121,8 +121,8 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.Invoker.*;
 import static org.apache.hadoop.fs.s3a.S3AUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index c54d3e26..440739d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -24,7 +24,7 @@ import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.SSECustomerKey;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.CanSetReadahead;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a01a80..a5f7d75 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -35,7 +35,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
 
 import com.google.common.base.Preconditions;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 9939bb2..7f9e57e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index e959908..3b89bde 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.auth.AWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.net.URI;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
index 4b6a77e..fdaf9bd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
@@ -32,7 +32,7 @@ import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
index 85cc38a..596dd95 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
@@ -33,7 +33,7 @@ import java.util.Map;
 import com.amazonaws.services.s3.model.PartETag;
 import com.google.common.base.Preconditions;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
index 6cf1f1e..cf84cb3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
@@ -28,7 +28,7 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
index cf365c2..a619fc7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
@@ -28,7 +28,7 @@ import com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.WriteOperationHelper;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
index a4d39d7..d5d256a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
@@ -30,7 +30,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
index 66ada49..91e64cd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
@@ -26,7 +26,7 @@ import com.amazonaws.regions.Regions;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[05/50] [abbrv] hadoop git commit: YARN-8326. Removed exit code file check for launched container. Contributed by Shane Kumpf

Posted by gi...@apache.org.
YARN-8326.  Removed exit code file check for launched container.
            Contributed by Shane Kumpf


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a32bc39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a32bc39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a32bc39

Branch: refs/heads/HADOOP-15461
Commit: 8a32bc39eb210fca8052c472601e24c2446b4cc2
Parents: 1cdce86
Author: Eric Yang <ey...@apache.org>
Authored: Fri Jun 22 19:12:48 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Jun 22 19:12:48 2018 -0400

----------------------------------------------------------------------
 .../launcher/ContainerLaunch.java                | 19 -------------------
 1 file changed, 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a32bc39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index bb842af..04295e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -808,25 +808,6 @@ public class ContainerLaunch implements Callable<Integer> {
       }
     }
 
-    final int sleepMsec = 100;
-    int msecLeft = 2000;
-    if (pidFilePath != null) {
-      File file = new File(getExitCodeFile(pidFilePath.toString()));
-      while (!file.exists() && msecLeft >= 0) {
-        try {
-          Thread.sleep(sleepMsec);
-        } catch (InterruptedException e) {
-        }
-        msecLeft -= sleepMsec;
-      }
-      if (msecLeft < 0) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Timeout while waiting for the exit code file:  "
-              + file.getAbsolutePath());
-        }
-      }
-    }
-
     // Reap the container
     boolean result = exec.reapContainer(
         new ContainerReapContext.Builder()


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[41/50] [abbrv] hadoop git commit: HDDS-185: TestCloseContainerByPipeline#testCloseContainerViaRatis fail intermittently. Contributed by Shashikant Banerjee.

Posted by gi...@apache.org.
HDDS-185: TestCloseContainerByPipeline#testCloseContainerViaRatis fail intermittently. Contributed by Shashikant Banerjee.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/852ca6b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/852ca6b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/852ca6b4

Branch: refs/heads/HADOOP-15461
Commit: 852ca6b432a801df51f957a9b169483008e3c744
Parents: 2911943
Author: Nanda kumar <na...@apache.org>
Authored: Fri Jun 29 04:11:39 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Jun 29 04:11:39 2018 +0530

----------------------------------------------------------------------
 .../TestCloseContainerByPipeline.java           | 65 +++++++++++++++++---
 1 file changed, 58 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/852ca6b4/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index 9e8cb46..265c82b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -33,6 +35,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
 import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
 import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
 import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
@@ -45,6 +48,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeoutException;
 
 public class TestCloseContainerByPipeline {
@@ -89,6 +93,59 @@ public class TestCloseContainerByPipeline {
   }
 
   @Test
+  public void testIfCloseContainerCommandHandlerIsInvoked() throws Exception {
+    OzoneOutputStream key = objectStore.getVolume("test").getBucket("test")
+        .createKey("testCloseContainer", 1024, ReplicationType.STAND_ALONE,
+            ReplicationFactor.ONE);
+    key.write("standalone".getBytes());
+    key.close();
+
+    //get the name of a valid container
+    KsmKeyArgs keyArgs =
+        new KsmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
+            .setType(HddsProtos.ReplicationType.STAND_ALONE)
+            .setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
+            .setKeyName("testCloseContainer").build();
+
+    KsmKeyLocationInfo ksmKeyLocationInfo =
+        cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
+            .get(0).getBlocksLatestVersionOnly().get(0);
+
+    long containerID = ksmKeyLocationInfo.getContainerID();
+    List<DatanodeDetails> datanodes =
+        cluster.getStorageContainerManager().getContainerInfo(containerID)
+            .getPipeline().getMachines();
+    Assert.assertTrue(datanodes.size() == 1);
+
+    DatanodeDetails datanodeDetails = datanodes.get(0);
+    HddsDatanodeService datanodeService = null;
+    Assert
+        .assertFalse(isContainerClosed(cluster, containerID, datanodeDetails));
+    for (HddsDatanodeService datanodeServiceItr : cluster.getHddsDatanodes()) {
+      if (datanodeDetails.equals(datanodeServiceItr.getDatanodeDetails())) {
+        datanodeService = datanodeServiceItr;
+        break;
+      }
+    }
+    CommandHandler closeContainerHandler =
+        datanodeService.getDatanodeStateMachine().getCommandDispatcher()
+            .getCloseContainerHandler();
+    int lastInvocationCount = closeContainerHandler.getInvocationCount();
+    //send the order to close the container
+    cluster.getStorageContainerManager().getScmNodeManager()
+        .addDatanodeCommand(datanodeDetails.getUuid(),
+            new CloseContainerCommand(containerID,
+                HddsProtos.ReplicationType.STAND_ALONE));
+    GenericTestUtils
+        .waitFor(() -> isContainerClosed(cluster, containerID, datanodeDetails),
+            500, 5 * 1000);
+    // Make sure the closeContainerCommandHandler is Invoked
+    Assert.assertTrue(
+        closeContainerHandler.getInvocationCount() > lastInvocationCount);
+
+  }
+
+  @Test
   public void testCloseContainerViaStandaAlone()
       throws IOException, TimeoutException, InterruptedException {
 
@@ -205,13 +262,7 @@ public class TestCloseContainerByPipeline {
           containerData =
               datanodeService.getDatanodeStateMachine().getContainer()
                   .getContainerManager().readContainer(containerID);
-          if (!containerData.isOpen()) {
-            // make sure the closeContainerHandler on the Datanode is invoked
-            Assert.assertTrue(
-                datanodeService.getDatanodeStateMachine().getCommandDispatcher()
-                    .getCloseContainerHandler().getInvocationCount() > 0);
-            return true;
-          }
+          return !containerData.isOpen();
         }
     } catch (StorageContainerException e) {
       throw new AssertionError(e);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[14/50] [abbrv] hadoop git commit: HDDS-191. Queue SCMCommands via EventQueue in SCM. Contributed by Elek, Marton.

Posted by gi...@apache.org.
HDDS-191. Queue SCMCommands via EventQueue in SCM.
Contributed by Elek, Marton.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d6bba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d6bba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d6bba

Branch: refs/heads/HADOOP-15461
Commit: a55d6bba71c81c1c4e9d8cd11f55c78f10a548b0
Parents: 4ffe68a
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Jun 25 13:05:22 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Jun 25 13:05:22 2018 -0700

----------------------------------------------------------------------
 .../protocol/commands/CommandForDatanode.java   | 45 ++++++++++++++++++++
 .../hadoop/hdds/scm/node/SCMNodeManager.java    | 20 ++++++++-
 .../scm/server/StorageContainerManager.java     |  8 +++-
 .../hadoop/hdds/scm/node/TestNodeManager.java   | 39 +++++++++++++++++
 4 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
new file mode 100644
index 0000000..0c4964a
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.UUID;
+
+import com.google.protobuf.GeneratedMessage;
+
+/**
+ * Command for the datanode with the destination address.
+ */
+public class CommandForDatanode<T extends GeneratedMessage> {
+
+  private final UUID datanodeId;
+
+  private final SCMCommand<T> command;
+
+  public CommandForDatanode(UUID datanodeId, SCMCommand<T> command) {
+    this.datanodeId = datanodeId;
+    this.command = command;
+  }
+
+  public UUID getDatanodeId() {
+    return datanodeId;
+  }
+
+  public SCMCommand<T> getCommand() {
+    return command;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index b339fb7..fc8b013 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.scm.VersionInfo;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -42,11 +46,14 @@ import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
 import org.apache.hadoop.ozone.protocol.VersionResponse;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
 import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
 import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.protobuf.GeneratedMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -100,7 +107,8 @@ import static org.apache.hadoop.util.Time.monotonicNow;
  * as soon as you read it.
  */
 public class SCMNodeManager
-    implements NodeManager, StorageContainerNodeProtocol {
+    implements NodeManager, StorageContainerNodeProtocol,
+    EventHandler<CommandForDatanode> {
 
   @VisibleForTesting
   static final Logger LOG =
@@ -154,6 +162,9 @@ public class SCMNodeManager
   private final SCMNodePoolManager nodePoolManager;
   private final StorageContainerManager scmManager;
 
+  public static final Event<CommandForDatanode> DATANODE_COMMAND =
+      new TypedEvent<>(CommandForDatanode.class, "DATANODE_COMMAND");
+
   /**
    * Constructs SCM machine Manager.
    */
@@ -871,4 +882,11 @@ public class SCMNodeManager
   public void setStaleNodeIntervalMs(long interval) {
     this.staleNodeIntervalMs = interval;
   }
+
+  @Override
+  public void onMessage(CommandForDatanode commandForDatanode,
+      EventPublisher publisher) {
+    addDatanodeCommand(commandForDatanode.getDatanodeId(),
+        commandForDatanode.getCommand());
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 78f13cb..5725d23 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
 import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
+import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RPC;
@@ -51,6 +52,7 @@ import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.common.Storage.StorageState;
 import org.apache.hadoop.ozone.common.StorageInfo;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.StringUtils;
@@ -161,8 +163,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
       throw new SCMException("SCM not initialized.", ResultCodes
           .SCM_NOT_INITIALIZED);
     }
+    EventQueue eventQueue = new EventQueue();
+
+    SCMNodeManager nm = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
+    scmNodeManager = nm;
+    eventQueue.addHandler(SCMNodeManager.DATANODE_COMMAND, nm);
 
-    scmNodeManager = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
     scmContainerManager = new ContainerMapping(conf, getScmNodeManager(),
         cacheSize);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 2b04d6b..824a135 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
 
 import com.google.common.base.Supplier;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -29,7 +30,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
 import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.test.PathUtils;
@@ -1165,4 +1169,39 @@ public class TestNodeManager {
       assertEquals(expectedRemaining, foundRemaining);
     }
   }
+
+  @Test
+  public void testHandlingSCMCommandEvent() {
+    OzoneConfiguration conf = getConf();
+    conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+        100, TimeUnit.MILLISECONDS);
+
+    DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+    String dnId = datanodeDetails.getUuidString();
+    String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+    List<StorageReportProto> reports =
+        TestUtils.createStorageReport(100, 10, 90,
+            storagePath, null, dnId, 1);
+
+    EventQueue eq = new EventQueue();
+    try (SCMNodeManager nodemanager = createNodeManager(conf)) {
+      eq.addHandler(SCMNodeManager.DATANODE_COMMAND, nodemanager);
+
+      nodemanager
+          .register(datanodeDetails, TestUtils.createNodeReport(reports));
+      eq.fireEvent(SCMNodeManager.DATANODE_COMMAND,
+          new CommandForDatanode(datanodeDetails.getUuid(),
+              new CloseContainerCommand(1L, ReplicationType.STAND_ALONE)));
+
+      eq.processAll(1000L);
+      List<SCMCommand> command =
+          nodemanager.sendHeartbeat(datanodeDetails, null);
+      Assert.assertEquals(1, command.size());
+      Assert
+          .assertEquals(command.get(0).getClass(), CloseContainerCommand.class);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[28/50] [abbrv] hadoop git commit: Revert "HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"

Posted by gi...@apache.org.
Revert "HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"

This reverts commit aaf03cc459a34af284f9735453aefd4ddb430d67.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d6fe5f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d6fe5f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d6fe5f3

Branch: refs/heads/HADOOP-15461
Commit: 0d6fe5f36be5b19aab89d995866e526c5feec758
Parents: aaf03cc
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:25:45 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:25:45 2018 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hdds/scm/ScmConfigKeys.java   |  26 ++
 .../org/apache/hadoop/ozone/OzoneConsts.java    |   1 +
 .../common/src/main/resources/ozone-default.xml |  47 +++
 .../container/replication/ReplicationQueue.java |  78 -----
 .../replication/ReplicationReqMsg.java          | 107 ------
 .../container/replication/package-info.java     |  23 --
 .../replication/TestReplicationQueue.java       | 134 --------
 .../container/replication/package-info.java     |  23 --
 .../hdds/scm/container/ContainerMapping.java    |  10 +-
 .../replication/ContainerSupervisor.java        | 340 +++++++++++++++++++
 .../container/replication/InProgressPool.java   | 255 ++++++++++++++
 .../scm/container/replication/PeriodicPool.java | 119 +++++++
 .../scm/container/replication/package-info.java |  23 ++
 .../hadoop/hdds/scm/node/NodeManager.java       |   6 +
 .../hadoop/hdds/scm/node/NodePoolManager.java   |  71 ++++
 .../hadoop/hdds/scm/node/SCMNodeManager.java    |  23 ++
 .../hdds/scm/node/SCMNodePoolManager.java       | 269 +++++++++++++++
 .../hdds/scm/container/MockNodeManager.java     |   6 +
 .../hdds/scm/node/TestSCMNodePoolManager.java   | 160 +++++++++
 .../testutils/ReplicationNodeManagerMock.java   |   5 +
 .../ReplicationNodePoolManagerMock.java         | 133 ++++++++
 .../hadoop/ozone/scm/TestContainerSQLCli.java   |  31 ++
 .../org/apache/hadoop/ozone/scm/cli/SQLCLI.java |  74 ++++
 23 files changed, 1596 insertions(+), 368 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index df6fbf0..85407e6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,6 +243,32 @@ public final class ScmConfigKeys {
   public static final String
       OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
 
+  /**
+   * Don't start processing a pool if we have not had a minimum number of
+   * seconds from the last processing.
+   */
+  public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
+      "ozone.scm.container.report.processing.interval";
+  public static final String
+      OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
+
+  /**
+   * This determines the total number of pools to be processed in parallel.
+   */
+  public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
+      "ozone.scm.max.nodepool.processing.threads";
+  public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
+  /**
+   * These 2 settings control the number of threads in executor pool and time
+   * outs for thw container reports from all nodes.
+   */
+  public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
+      "ozone.scm.max.container.report.threads";
+  public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
+  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
+      "ozone.scm.container.reports.wait.timeout";
+  public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
+      "5m";
 
   public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
       "ozone.scm.block.deletion.max.retry";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 08a5ffd..c40dc8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,6 +91,7 @@ public final class OzoneConsts {
   public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
   public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
   public static final String BLOCK_DB = "block.db";
+  public static final String NODEPOOL_DB = "nodepool.db";
   public static final String OPEN_CONTAINERS_DB = "openContainers.db";
   public static final String DELETED_BLOCK_DB = "deletedBlock.db";
   public static final String KSM_DB_NAME = "ksm.db";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 25365c8..7a91610 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,6 +572,25 @@
     </description>
   </property>
   <property>
+    <name>ozone.scm.container.report.processing.interval</name>
+    <value>60s</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>Time interval for scm to process container reports
+      for a node pool. Scm handles node pool reports in a cyclic clock
+      manner, it fetches pools periodically with this time interval.
+    </description>
+  </property>
+  <property>
+    <name>ozone.scm.container.reports.wait.timeout</name>
+    <value>300s</value>
+    <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+    <description>Maximum time to wait in seconds for processing all container
+      reports from
+      a node pool. It determines the timeout for a
+      node pool report.
+    </description>
+  </property>
+  <property>
     <name>ozone.scm.container.size.gb</name>
     <value>5</value>
     <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -774,6 +793,17 @@
     </description>
   </property>
   <property>
+    <name>ozone.scm.max.container.report.threads</name>
+    <value>100</value>
+    <tag>OZONE, PERFORMANCE</tag>
+    <description>
+      Maximum number of threads to process container reports in scm.
+      Each container report from a data node is processed by scm in a worker
+      thread, fetched from a thread pool. This property is used to control the
+      maximum size of the thread pool.
+    </description>
+  </property>
+  <property>
     <name>ozone.scm.max.hb.count.to.process</name>
     <value>5000</value>
     <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -785,6 +815,14 @@
     </description>
   </property>
   <property>
+    <name>ozone.scm.max.nodepool.processing.threads</name>
+    <value>1</value>
+    <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+    <description>
+      Number of node pools to process in parallel.
+    </description>
+  </property>
+  <property>
     <name>ozone.scm.names</name>
     <value/>
     <tag>OZONE</tag>
@@ -806,6 +844,15 @@
     </description>
   </property>
   <property>
+    <name>ozone.scm.max.nodepool.processing.threads</name>
+    <value>1</value>
+    <tag>OZONE, SCM</tag>
+    <description>
+      Controls the number of node pools that can be processed in parallel by
+      Container Supervisor.
+    </description>
+  </property>
+  <property>
     <name>ozone.trace.enabled</name>
     <value>false</value>
     <tag>OZONE, DEBUG</tag>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
deleted file mode 100644
index b83ecf1..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Queue;
-
-/**
- * Priority queue to handle under-replicated and over replicated containers
- * in ozone. ReplicationManager will consume these messages and decide
- * accordingly.
- */
-public class ReplicationQueue {
-
-  private final Queue<ReplicationReqMsg> queue;
-
-  ReplicationQueue() {
-    queue = new PriorityQueue<>();
-  }
-
-  public synchronized boolean add(ReplicationReqMsg repObj) {
-    if (this.queue.contains(repObj)) {
-      // Remove the earlier message and insert this one
-      this.queue.remove(repObj);
-      return this.queue.add(repObj);
-    } else {
-      return this.queue.add(repObj);
-    }
-  }
-
-  public synchronized boolean remove(ReplicationReqMsg repObj) {
-    return queue.remove(repObj);
-  }
-
-  /**
-   * Retrieves, but does not remove, the head of this queue,
-   * or returns {@code null} if this queue is empty.
-   *
-   * @return the head of this queue, or {@code null} if this queue is empty
-   */
-  public synchronized ReplicationReqMsg peek() {
-    return queue.peek();
-  }
-
-  /**
-   * Retrieves and removes the head of this queue,
-   * or returns {@code null} if this queue is empty.
-   *
-   * @return the head of this queue, or {@code null} if this queue is empty
-   */
-  public synchronized ReplicationReqMsg poll() {
-    return queue.poll();
-  }
-
-  public synchronized boolean removeAll(List<ReplicationReqMsg> repObjs) {
-    return queue.removeAll(repObjs);
-  }
-
-  public int size() {
-    return queue.size();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
deleted file mode 100644
index 8d26fc3..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.io.Serializable;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.math.NumberUtils;
-
-/**
- * Wrapper class for hdds replication queue. Implements its natural
- * ordering for priority queue.
- */
-public class ReplicationReqMsg implements Comparable<ReplicationReqMsg>,
-    Serializable {
-  private final long containerId;
-  private final short replicationCount;
-  private final short expecReplicationCount;
-  private final long timestamp;
-
-  public ReplicationReqMsg(long containerId, short replicationCount,
-      long timestamp, short expecReplicationCount) {
-    this.containerId = containerId;
-    this.replicationCount = replicationCount;
-    this.timestamp = timestamp;
-    this.expecReplicationCount = expecReplicationCount;
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less
-   * than, equal to, or greater than the specified object.
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object
-   * is less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(ReplicationReqMsg o) {
-    if (this == o) {
-      return 0;
-    }
-    if (o == null) {
-      return 1;
-    }
-    int retVal = NumberUtils
-        .compare(getReplicationCount() - getExpecReplicationCount(),
-            o.getReplicationCount() - o.getExpecReplicationCount());
-    if (retVal != 0) {
-      return retVal;
-    }
-    return NumberUtils.compare(getTimestamp(), o.getTimestamp());
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(91, 1011)
-        .append(getContainerId())
-        .toHashCode();
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    ReplicationReqMsg that = (ReplicationReqMsg) o;
-    return new EqualsBuilder().append(getContainerId(), that.getContainerId())
-        .isEquals();
-  }
-
-  public long getContainerId() {
-    return containerId;
-  }
-
-  public short getReplicationCount() {
-    return replicationCount;
-  }
-
-  public long getTimestamp() {
-    return timestamp;
-  }
-
-  public short getExpecReplicationCount() {
-    return expecReplicationCount;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
deleted file mode 100644
index 7f335e3..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-/**
- * Ozone Container replicaton related classes.
- */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
deleted file mode 100644
index 39c61d3..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.util.Random;
-import java.util.UUID;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for ReplicationQueue.
- */
-public class TestReplicationQueue {
-
-  private ReplicationQueue replicationQueue;
-  private Random random;
-
-  @Before
-  public void setUp() {
-    replicationQueue = new ReplicationQueue();
-    random = new Random();
-  }
-
-  @Test
-  public void testDuplicateAddOp() {
-    long contId = random.nextLong();
-    String nodeId = UUID.randomUUID().toString();
-    ReplicationReqMsg obj1, obj2, obj3;
-    long time = Time.monotonicNow();
-    obj1 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
-    obj2 = new ReplicationReqMsg(contId, (short) 2, time + 1, (short) 3);
-    obj3 = new ReplicationReqMsg(contId, (short) 1, time+2, (short) 3);
-
-    replicationQueue.add(obj1);
-    replicationQueue.add(obj2);
-    replicationQueue.add(obj3);
-    Assert.assertEquals("Should add only 1 msg as second one is duplicate",
-        1, replicationQueue.size());
-    ReplicationReqMsg temp = replicationQueue.poll();
-    Assert.assertEquals(temp, obj3);
-  }
-
-  @Test
-  public void testPollOp() {
-    long contId = random.nextLong();
-    String nodeId = UUID.randomUUID().toString();
-    ReplicationReqMsg msg1, msg2, msg3, msg4, msg5;
-    msg1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
-        (short) 3);
-    long time = Time.monotonicNow();
-    msg2 = new ReplicationReqMsg(contId + 1, (short) 4, time, (short) 3);
-    msg3 = new ReplicationReqMsg(contId + 2, (short) 0, time, (short) 3);
-    msg4 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
-    // Replication message for same container but different nodeId
-    msg5 = new ReplicationReqMsg(contId + 1, (short) 2, time, (short) 3);
-
-    replicationQueue.add(msg1);
-    replicationQueue.add(msg2);
-    replicationQueue.add(msg3);
-    replicationQueue.add(msg4);
-    replicationQueue.add(msg5);
-    Assert.assertEquals("Should have 3 objects",
-        3, replicationQueue.size());
-
-    // Since Priority queue orders messages according to replication count,
-    // message with lowest replication should be first
-    ReplicationReqMsg temp;
-    temp = replicationQueue.poll();
-    Assert.assertEquals("Should have 2 objects",
-        2, replicationQueue.size());
-    Assert.assertEquals(temp, msg3);
-
-    temp = replicationQueue.poll();
-    Assert.assertEquals("Should have 1 objects",
-        1, replicationQueue.size());
-    Assert.assertEquals(temp, msg5);
-
-    // Message 2 should be ordered before message 5 as both have same replication
-    // number but message 2 has earlier timestamp.
-    temp = replicationQueue.poll();
-    Assert.assertEquals("Should have 0 objects",
-        replicationQueue.size(), 0);
-    Assert.assertEquals(temp, msg4);
-  }
-
-  @Test
-  public void testRemoveOp() {
-    long contId = random.nextLong();
-    String nodeId = UUID.randomUUID().toString();
-    ReplicationReqMsg obj1, obj2, obj3;
-    obj1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
-        (short) 3);
-    obj2 = new ReplicationReqMsg(contId + 1, (short) 2, Time.monotonicNow(),
-        (short) 3);
-    obj3 = new ReplicationReqMsg(contId + 2, (short) 3, Time.monotonicNow(),
-        (short) 3);
-
-    replicationQueue.add(obj1);
-    replicationQueue.add(obj2);
-    replicationQueue.add(obj3);
-    Assert.assertEquals("Should have 3 objects",
-        3, replicationQueue.size());
-
-    replicationQueue.remove(obj3);
-    Assert.assertEquals("Should have 2 objects",
-        2, replicationQueue.size());
-
-    replicationQueue.remove(obj2);
-    Assert.assertEquals("Should have 1 objects",
-        1, replicationQueue.size());
-
-    replicationQueue.remove(obj1);
-    Assert.assertEquals("Should have 0 objects",
-        0, replicationQueue.size());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
deleted file mode 100644
index 5b1fd0f..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * SCM Testing and Mocking Utils.
- */
-package org.apache.hadoop.ozone.container.replication;
-// Test classes for Replication functionality.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 9fd30f2..b563e90 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -79,6 +80,7 @@ public class ContainerMapping implements Mapping {
   private final PipelineSelector pipelineSelector;
   private final ContainerStateManager containerStateManager;
   private final LeaseManager<ContainerInfo> containerLeaseManager;
+  private final ContainerSupervisor containerSupervisor;
   private final float containerCloseThreshold;
   private final ContainerCloser closer;
   private final long size;
@@ -125,7 +127,9 @@ public class ContainerMapping implements Mapping {
         OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
     this.containerStateManager =
         new ContainerStateManager(conf, this);
-
+    this.containerSupervisor =
+        new ContainerSupervisor(conf, nodeManager,
+            nodeManager.getNodePoolManager());
     this.containerCloseThreshold = conf.getFloat(
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
         ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -403,8 +407,8 @@ public class ContainerMapping implements Mapping {
       throws IOException {
     List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
         containerInfos = reports.getReportsList();
-
-     for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+    containerSupervisor.handleContainerReport(datanodeDetails, reports);
+    for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
         containerInfos) {
       byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
       lock.lock();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
new file mode 100644
index 0000000..5bd0574
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static com.google.common.util.concurrent.Uninterruptibles
+    .sleepUninterruptibly;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
+
+/**
+ * This class takes a set of container reports that belong to a pool and then
+ * computes the replication levels for each container.
+ */
+public class ContainerSupervisor implements Closeable {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(ContainerSupervisor.class);
+
+  private final NodePoolManager poolManager;
+  private final HashSet<String> poolNames;
+  private final PriorityQueue<PeriodicPool> poolQueue;
+  private final NodeManager nodeManager;
+  private final long containerProcessingLag;
+  private final AtomicBoolean runnable;
+  private final ExecutorService executorService;
+  private final long maxPoolWait;
+  private long poolProcessCount;
+  private final List<InProgressPool> inProgressPoolList;
+  private final AtomicInteger threadFaultCount;
+  private final int inProgressPoolMaxCount;
+
+  private final ReadWriteLock inProgressPoolListLock;
+
+  /**
+   * Returns the number of times we have processed pools.
+   * @return long
+   */
+  public long getPoolProcessCount() {
+    return poolProcessCount;
+  }
+
+
+  /**
+   * Constructs a class that computes Replication Levels.
+   *
+   * @param conf - OzoneConfiguration
+   * @param nodeManager - Node Manager
+   * @param poolManager - Pool Manager
+   */
+  public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
+                             NodePoolManager poolManager) {
+    Preconditions.checkNotNull(poolManager);
+    Preconditions.checkNotNull(nodeManager);
+    this.containerProcessingLag =
+        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
+            OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
+            TimeUnit.SECONDS
+        ) * 1000;
+    int maxContainerReportThreads =
+        conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
+            OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
+        );
+    this.maxPoolWait =
+        conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
+            OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
+            TimeUnit.MILLISECONDS);
+    this.inProgressPoolMaxCount = conf.getInt(
+        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
+        OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
+    this.poolManager = poolManager;
+    this.nodeManager = nodeManager;
+    this.poolNames = new HashSet<>();
+    this.poolQueue = new PriorityQueue<>();
+    this.runnable = new AtomicBoolean(true);
+    this.threadFaultCount = new AtomicInteger(0);
+    this.executorService = newCachedThreadPool(
+        new ThreadFactoryBuilder().setDaemon(true)
+            .setNameFormat("Container Reports Processing Thread - %d")
+            .build(), maxContainerReportThreads);
+    this.inProgressPoolList = new LinkedList<>();
+    this.inProgressPoolListLock = new ReentrantReadWriteLock();
+
+    initPoolProcessThread();
+  }
+
+  private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
+      int maxThreads) {
+    return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
+        new LinkedBlockingQueue<>(), threadFactory);
+  }
+
+  /**
+   * Returns the number of pools that are under process right now.
+   * @return  int - Number of pools that are in process.
+   */
+  public int getInProgressPoolCount() {
+    return inProgressPoolList.size();
+  }
+
+  /**
+   * Exits the background thread.
+   */
+  public void setExit() {
+    this.runnable.set(false);
+  }
+
+  /**
+   * Adds or removes pools from names that we need to process.
+   *
+   * There are two different cases that we need to process.
+   * The case where some pools are being added and some times we have to
+   * handle cases where pools are removed.
+   */
+  private void refreshPools() {
+    List<String> pools = this.poolManager.getNodePools();
+    if (pools != null) {
+
+      HashSet<String> removedPools =
+          computePoolDifference(this.poolNames, new HashSet<>(pools));
+
+      HashSet<String> addedPools =
+          computePoolDifference(new HashSet<>(pools), this.poolNames);
+      // TODO: Support remove pool API in pool manager so that this code
+      // path can be tested. This never happens in the current code base.
+      for (String poolName : removedPools) {
+        for (PeriodicPool periodicPool : poolQueue) {
+          if (periodicPool.getPoolName().compareTo(poolName) == 0) {
+            poolQueue.remove(periodicPool);
+          }
+        }
+      }
+      // Remove the pool names that we have in the list.
+      this.poolNames.removeAll(removedPools);
+
+      for (String poolName : addedPools) {
+        poolQueue.add(new PeriodicPool(poolName));
+      }
+
+      // Add to the pool names we are tracking.
+      poolNames.addAll(addedPools);
+    }
+
+  }
+
+  /**
+   * Handle the case where pools are added.
+   *
+   * @param newPools - New Pools list
+   * @param oldPool - oldPool List.
+   */
+  private HashSet<String> computePoolDifference(HashSet<String> newPools,
+      Set<String> oldPool) {
+    Preconditions.checkNotNull(newPools);
+    Preconditions.checkNotNull(oldPool);
+    HashSet<String> newSet = new HashSet<>(newPools);
+    newSet.removeAll(oldPool);
+    return newSet;
+  }
+
+  private void initPoolProcessThread() {
+
+    /*
+     * Task that runs to check if we need to start a pool processing job.
+     * if so we create a pool reconciliation job and find out of all the
+     * expected containers are on the nodes.
+     */
+    Runnable processPools = () -> {
+      while (runnable.get()) {
+        // Make sure that we don't have any new pools.
+        refreshPools();
+        while (inProgressPoolList.size() < inProgressPoolMaxCount) {
+          PeriodicPool pool = poolQueue.poll();
+          if (pool != null) {
+            if (pool.getLastProcessedTime() + this.containerProcessingLag >
+                Time.monotonicNow()) {
+              LOG.debug("Not within the time window for processing: {}",
+                  pool.getPoolName());
+              // we might over sleep here, not a big deal.
+              sleepUninterruptibly(this.containerProcessingLag,
+                  TimeUnit.MILLISECONDS);
+            }
+            LOG.debug("Adding pool {} to container processing queue",
+                pool.getPoolName());
+            InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
+                pool, this.nodeManager, this.poolManager, this.executorService);
+            inProgressPool.startReconciliation();
+            inProgressPoolListLock.writeLock().lock();
+            try {
+              inProgressPoolList.add(inProgressPool);
+            } finally {
+              inProgressPoolListLock.writeLock().unlock();
+            }
+            poolProcessCount++;
+          } else {
+            break;
+          }
+        }
+        sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
+        inProgressPoolListLock.readLock().lock();
+        try {
+          for (InProgressPool inProgressPool : inProgressPoolList) {
+            inProgressPool.finalizeReconciliation();
+            poolQueue.add(inProgressPool.getPool());
+          }
+        } finally {
+          inProgressPoolListLock.readLock().unlock();
+        }
+        inProgressPoolListLock.writeLock().lock();
+        try {
+          inProgressPoolList.clear();
+        } finally {
+          inProgressPoolListLock.writeLock().unlock();
+        }
+      }
+    };
+
+    // We will have only one thread for pool processing.
+    Thread poolProcessThread = new Thread(processPools);
+    poolProcessThread.setDaemon(true);
+    poolProcessThread.setName("Pool replica thread");
+    poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
+      // Let us just restart this thread after logging a critical error.
+      // if this thread is not running we cannot handle commands from SCM.
+      LOG.error("Critical Error : Pool replica thread encountered an " +
+          "error. Thread: {} Error Count : {}", t.toString(), e,
+          threadFaultCount.incrementAndGet());
+      poolProcessThread.start();
+      // TODO : Add a config to restrict how many times we will restart this
+      // thread in a single session.
+    });
+    poolProcessThread.start();
+  }
+
+  /**
+   * Adds a container report to appropriate inProgress Pool.
+   * @param containerReport  -- Container report for a specific container from
+   * a datanode.
+   */
+  public void handleContainerReport(DatanodeDetails datanodeDetails,
+      ContainerReportsProto containerReport) {
+    inProgressPoolListLock.readLock().lock();
+    try {
+      String poolName = poolManager.getNodePool(datanodeDetails);
+      for (InProgressPool ppool : inProgressPoolList) {
+        if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
+          ppool.handleContainerReport(datanodeDetails, containerReport);
+          return;
+        }
+      }
+      // TODO: Decide if we can do anything else with this report.
+      LOG.debug("Discarding the container report for pool {}. " +
+              "That pool is not currently in the pool reconciliation process." +
+              " Container Name: {}", poolName, datanodeDetails);
+    } catch (SCMException e) {
+      LOG.warn("Skipping processing container report from datanode {}, "
+              + "cause: failed to get the corresponding node pool",
+          datanodeDetails.toString(), e);
+    } finally {
+      inProgressPoolListLock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Get in process pool list, used for testing.
+   * @return List of InProgressPool
+   */
+  @VisibleForTesting
+  public List<InProgressPool> getInProcessPoolList() {
+    return inProgressPoolList;
+  }
+
+  /**
+   * Shutdown the Container Replication Manager.
+   * @throws IOException if an I/O error occurs
+   */
+  @Override
+  public void close() throws IOException {
+    setExit();
+    HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
new file mode 100644
index 0000000..4b54731
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerInfo;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * These are pools that are actively checking for replication status of the
+ * containers.
+ */
+public final class InProgressPool {
+  public static final Logger LOG =
+      LoggerFactory.getLogger(InProgressPool.class);
+
+  private final PeriodicPool pool;
+  private final NodeManager nodeManager;
+  private final NodePoolManager poolManager;
+  private final ExecutorService executorService;
+  private final Map<Long, Integer> containerCountMap;
+  private final Map<UUID, Boolean> processedNodeSet;
+  private final long startTime;
+  private ProgressStatus status;
+  private AtomicInteger nodeCount;
+  private AtomicInteger nodeProcessed;
+  private AtomicInteger containerProcessedCount;
+  private long maxWaitTime;
+  /**
+   * Constructs an pool that is being processed.
+   *  @param maxWaitTime - Maximum wait time in milliseconds.
+   * @param pool - Pool that we are working against
+   * @param nodeManager - Nodemanager
+   * @param poolManager - pool manager
+   * @param executorService - Shared Executor service.
+   */
+  InProgressPool(long maxWaitTime, PeriodicPool pool,
+      NodeManager nodeManager, NodePoolManager poolManager,
+                 ExecutorService executorService) {
+    Preconditions.checkNotNull(pool);
+    Preconditions.checkNotNull(nodeManager);
+    Preconditions.checkNotNull(poolManager);
+    Preconditions.checkNotNull(executorService);
+    Preconditions.checkArgument(maxWaitTime > 0);
+    this.pool = pool;
+    this.nodeManager = nodeManager;
+    this.poolManager = poolManager;
+    this.executorService = executorService;
+    this.containerCountMap = new ConcurrentHashMap<>();
+    this.processedNodeSet = new ConcurrentHashMap<>();
+    this.maxWaitTime = maxWaitTime;
+    startTime = Time.monotonicNow();
+  }
+
+  /**
+   * Returns periodic pool.
+   *
+   * @return PeriodicPool
+   */
+  public PeriodicPool getPool() {
+    return pool;
+  }
+
+  /**
+   * We are done if we have got reports from all nodes or we have
+   * done waiting for the specified time.
+   *
+   * @return true if we are done, false otherwise.
+   */
+  public boolean isDone() {
+    return (nodeCount.get() == nodeProcessed.get()) ||
+        (this.startTime + this.maxWaitTime) > Time.monotonicNow();
+  }
+
+  /**
+   * Gets the number of containers processed.
+   *
+   * @return int
+   */
+  public int getContainerProcessedCount() {
+    return containerProcessedCount.get();
+  }
+
+  /**
+   * Returns the start time in milliseconds.
+   *
+   * @return - Start Time.
+   */
+  public long getStartTime() {
+    return startTime;
+  }
+
+  /**
+   * Get the number of nodes in this pool.
+   *
+   * @return - node count
+   */
+  public int getNodeCount() {
+    return nodeCount.get();
+  }
+
+  /**
+   * Get the number of nodes that we have already processed container reports
+   * from.
+   *
+   * @return - Processed count.
+   */
+  public int getNodeProcessed() {
+    return nodeProcessed.get();
+  }
+
+  /**
+   * Returns the current status.
+   *
+   * @return Status
+   */
+  public ProgressStatus getStatus() {
+    return status;
+  }
+
+  /**
+   * Starts the reconciliation process for all the nodes in the pool.
+   */
+  public void startReconciliation() {
+    List<DatanodeDetails> datanodeDetailsList =
+        this.poolManager.getNodes(pool.getPoolName());
+    if (datanodeDetailsList.size() == 0) {
+      LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
+          pool.getPoolName());
+      this.status = ProgressStatus.Error;
+      return;
+    }
+
+    nodeProcessed = new AtomicInteger(0);
+    containerProcessedCount = new AtomicInteger(0);
+    nodeCount = new AtomicInteger(0);
+    this.status = ProgressStatus.InProgress;
+    this.getPool().setLastProcessedTime(Time.monotonicNow());
+  }
+
+  /**
+   * Queues a container Report for handling. This is done in a worker thread
+   * since decoding a container report might be compute intensive . We don't
+   * want to block since we have asked for bunch of container reports
+   * from a set of datanodes.
+   *
+   * @param containerReport - ContainerReport
+   */
+  public void handleContainerReport(DatanodeDetails datanodeDetails,
+      ContainerReportsProto containerReport) {
+    if (status == ProgressStatus.InProgress) {
+      executorService.submit(processContainerReport(datanodeDetails,
+          containerReport));
+    } else {
+      LOG.debug("Cannot handle container report when the pool is in {} status.",
+          status);
+    }
+  }
+
+  private Runnable processContainerReport(DatanodeDetails datanodeDetails,
+      ContainerReportsProto reports) {
+    return () -> {
+      if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
+          (k) -> true)) {
+        nodeProcessed.incrementAndGet();
+        LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
+            datanodeDetails.getUuid());
+        for (ContainerInfo info : reports.getReportsList()) {
+          containerProcessedCount.incrementAndGet();
+          LOG.debug("Total Containers processed: {} Container Name: {}",
+              containerProcessedCount.get(), info.getContainerID());
+
+          // Update the container map with count + 1 if the key exists or
+          // update the map with 1. Since this is a concurrentMap the
+          // computation and update is atomic.
+          containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
+        }
+      }
+    };
+  }
+
+  /**
+   * Filter the containers based on specific rules.
+   *
+   * @param predicate -- Predicate to filter by
+   * @return A list of map entries.
+   */
+  public List<Map.Entry<Long, Integer>> filterContainer(
+      Predicate<Map.Entry<Long, Integer>> predicate) {
+    return containerCountMap.entrySet().stream()
+        .filter(predicate).collect(Collectors.toList());
+  }
+
+  /**
+   * Used only for testing, calling this will abort container report
+   * processing. This is very dangerous call and should not be made by any users
+   */
+  @VisibleForTesting
+  public void setDoneProcessing() {
+    nodeProcessed.set(nodeCount.get());
+  }
+
+  /**
+   * Returns the pool name.
+   *
+   * @return Name of the pool.
+   */
+  String getPoolName() {
+    return pool.getPoolName();
+  }
+
+  public void finalizeReconciliation() {
+    status = ProgressStatus.Done;
+    //TODO: Add finalizing logic. This is where actual reconciliation happens.
+  }
+
+  /**
+   * Current status of the computing replication status.
+   */
+  public enum ProgressStatus {
+    InProgress, Done, Error
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
new file mode 100644
index 0000000..ef28aa7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Periodic pool is a pool with a time stamp, this allows us to process pools
+ * based on a cyclic clock.
+ */
+public class PeriodicPool implements Comparable<PeriodicPool> {
+  private final String poolName;
+  private long lastProcessedTime;
+  private AtomicLong totalProcessedCount;
+
+  /**
+   * Constructs a periodic pool.
+   *
+   * @param poolName - Name of the pool
+   */
+  public PeriodicPool(String poolName) {
+    this.poolName = poolName;
+    lastProcessedTime = 0;
+    totalProcessedCount = new AtomicLong(0);
+  }
+
+  /**
+   * Get pool Name.
+   * @return PoolName
+   */
+  public String getPoolName() {
+    return poolName;
+  }
+
+  /**
+   * Compares this object with the specified object for order.  Returns a
+   * negative integer, zero, or a positive integer as this object is less
+   * than, equal to, or greater than the specified object.
+   *
+   * @param o the object to be compared.
+   * @return a negative integer, zero, or a positive integer as this object is
+   * less than, equal to, or greater than the specified object.
+   * @throws NullPointerException if the specified object is null
+   * @throws ClassCastException   if the specified object's type prevents it
+   *                              from being compared to this object.
+   */
+  @Override
+  public int compareTo(PeriodicPool o) {
+    return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    PeriodicPool that = (PeriodicPool) o;
+
+    return poolName.equals(that.poolName);
+  }
+
+  @Override
+  public int hashCode() {
+    return poolName.hashCode();
+  }
+
+  /**
+   * Returns the Total Times we have processed this pool.
+   *
+   * @return processed count.
+   */
+  public long getTotalProcessedCount() {
+    return totalProcessedCount.get();
+  }
+
+  /**
+   * Gets the last time we processed this pool.
+   * @return time in milliseconds
+   */
+  public long getLastProcessedTime() {
+    return this.lastProcessedTime;
+  }
+
+
+  /**
+   * Sets the last processed time.
+   *
+   * @param lastProcessedTime - Long in milliseconds.
+   */
+
+  public void setLastProcessedTime(long lastProcessedTime) {
+    this.lastProcessedTime = lastProcessedTime;
+  }
+
+  /*
+   * Increments the total processed count.
+   */
+  public void incTotalProcessedCount() {
+    this.totalProcessedCount.incrementAndGet();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
new file mode 100644
index 0000000..7bbe2ef
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+/*
+ This package contains routines that manage replication of a container. This
+ relies on container reports to understand the replication level of a
+ container - UnderReplicated, Replicated, OverReplicated -- and manages the
+ replication level based on that.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 72d7e94..4392633 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,6 +124,12 @@ public interface NodeManager extends StorageContainerNodeProtocol,
   SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
 
   /**
+   * Returns the NodePoolManager associated with the NodeManager.
+   * @return NodePoolManager
+   */
+  NodePoolManager getNodePoolManager();
+
+  /**
    * Wait for the heartbeat is processed by NodeManager.
    * @return true if heartbeat has been processed.
    */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
new file mode 100644
index 0000000..46faf9ca
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Interface that defines SCM NodePoolManager.
+ */
+public interface NodePoolManager extends Closeable {
+
+  /**
+   * Add a node to a node pool.
+   * @param pool - name of the node pool.
+   * @param node - data node.
+   */
+  void addNode(String pool, DatanodeDetails node) throws IOException;
+
+  /**
+   * Remove a node from a node pool.
+   * @param pool - name of the node pool.
+   * @param node - data node.
+   * @throws SCMException
+   */
+  void removeNode(String pool, DatanodeDetails node)
+      throws SCMException;
+
+  /**
+   * Get a list of known node pools.
+   * @return a list of known node pool names or an empty list if not node pool
+   * is defined.
+   */
+  List<String> getNodePools();
+
+  /**
+   * Get all nodes of a node pool given the name of the node pool.
+   * @param pool - name of the node pool.
+   * @return a list of datanode ids or an empty list if the node pool was not
+   *  found.
+   */
+  List<DatanodeDetails> getNodes(String pool);
+
+  /**
+   * Get the node pool name if the node has been added to a node pool.
+   * @param datanodeDetails - datanode ID.
+   * @return node pool name if it has been assigned.
+   * null if the node has not been assigned to any node pool yet.
+   */
+  String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index adca8ea..fc8b013 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
+import com.google.protobuf.GeneratedMessage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -158,6 +159,7 @@ public class SCMNodeManager
   private ObjectName nmInfoBean;
 
   // Node pool manager.
+  private final SCMNodePoolManager nodePoolManager;
   private final StorageContainerManager scmManager;
 
   public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -208,6 +210,7 @@ public class SCMNodeManager
 
     registerMXBean();
 
+    this.nodePoolManager = new SCMNodePoolManager(conf);
     this.scmManager = scmManager;
   }
 
@@ -679,6 +682,7 @@ public class SCMNodeManager
   @Override
   public void close() throws IOException {
     unregisterMXBean();
+    nodePoolManager.close();
     executorService.shutdown();
     try {
       if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -756,6 +760,20 @@ public class SCMNodeManager
       LOG.info("Leaving startup chill mode.");
     }
 
+    // TODO: define node pool policy for non-default node pool.
+    // For now, all nodes are added to the "DefaultNodePool" upon registration
+    // if it has not been added to any node pool yet.
+    try {
+      if (nodePoolManager.getNodePool(datanodeDetails) == null) {
+        nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
+            datanodeDetails);
+      }
+    } catch (IOException e) {
+      // TODO: make sure registration failure is handled correctly.
+      return RegisteredCommand.newBuilder()
+          .setErrorCode(ErrorCode.errorNodeNotPermitted)
+          .build();
+    }
     // Updating Node Report, as registration is successful
     updateNodeStat(datanodeDetails.getUuid(), nodeReport);
     LOG.info("Data node with ID: {} Registered.",
@@ -842,6 +860,11 @@ public class SCMNodeManager
   }
 
   @Override
+  public NodePoolManager getNodePoolManager() {
+    return nodePoolManager;
+  }
+
+  @Override
   public Map<String, Integer> getNodeCount() {
     Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
     for(NodeState state : NodeState.values()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
new file mode 100644
index 0000000..faf330e
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+    .OZONE_SCM_DB_CACHE_SIZE_MB;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+    .FAILED_TO_FIND_NODE_IN_POOL;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+    .FAILED_TO_LOAD_NODEPOOL;
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
+
+/**
+ * SCM node pool manager that manges node pools.
+ */
+public final class SCMNodePoolManager implements NodePoolManager {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(SCMNodePoolManager.class);
+  private static final List<DatanodeDetails> EMPTY_NODE_LIST =
+      new ArrayList<>();
+  private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
+  public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
+
+  // DB that saves the node to node pool mapping.
+  private MetadataStore nodePoolStore;
+
+  // In-memory node pool to nodes mapping
+  private HashMap<String, Set<DatanodeDetails>> nodePools;
+
+  // Read-write lock for nodepool operations
+  private ReadWriteLock lock;
+
+  /**
+   * Construct SCMNodePoolManager class that manages node to node pool mapping.
+   * @param conf - configuration.
+   * @throws IOException
+   */
+  public SCMNodePoolManager(final OzoneConfiguration conf)
+      throws IOException {
+    final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
+        OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
+    File metaDir = getOzoneMetaDirPath(conf);
+    String scmMetaDataDir = metaDir.getPath();
+    File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
+    nodePoolStore = MetadataStoreBuilder.newBuilder()
+        .setConf(conf)
+        .setDbFile(nodePoolDBPath)
+        .setCacheSize(cacheSize * OzoneConsts.MB)
+        .build();
+    nodePools = new HashMap<>();
+    lock = new ReentrantReadWriteLock();
+    init();
+  }
+
+  /**
+   * Initialize the in-memory store based on persist store from level db.
+   * No lock is needed as init() is only invoked by constructor.
+   * @throws SCMException
+   */
+  private void init() throws SCMException {
+    try {
+      nodePoolStore.iterate(null, (key, value) -> {
+        try {
+          DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
+              HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
+          String poolName = DFSUtil.bytes2String(value);
+
+          Set<DatanodeDetails> nodePool = null;
+          if (nodePools.containsKey(poolName)) {
+            nodePool = nodePools.get(poolName);
+          } else {
+            nodePool = new HashSet<>();
+            nodePools.put(poolName, nodePool);
+          }
+          nodePool.add(nodeId);
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Adding node: {} to node pool: {}",
+                nodeId, poolName);
+          }
+        } catch (IOException e) {
+          LOG.warn("Can't add a datanode to node pool, continue next...");
+        }
+        return true;
+      });
+    } catch (IOException e) {
+      LOG.error("Loading node pool error " + e);
+      throw new SCMException("Failed to load node pool",
+          FAILED_TO_LOAD_NODEPOOL);
+    }
+  }
+
+  /**
+   * Add a datanode to a node pool.
+   * @param pool - name of the node pool.
+   * @param node - name of the datanode.
+   */
+  @Override
+  public void addNode(final String pool, final DatanodeDetails node)
+      throws IOException {
+    Preconditions.checkNotNull(pool, "pool name is null");
+    Preconditions.checkNotNull(node, "node is null");
+    lock.writeLock().lock();
+    try {
+      // add to the persistent store
+      nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
+          DFSUtil.string2Bytes(pool));
+
+      // add to the in-memory store
+      Set<DatanodeDetails> nodePool = null;
+      if (nodePools.containsKey(pool)) {
+        nodePool = nodePools.get(pool);
+      } else {
+        nodePool = new HashSet<DatanodeDetails>();
+        nodePools.put(pool, nodePool);
+      }
+      nodePool.add(node);
+    } finally {
+      lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Remove a datanode from a node pool.
+   * @param pool - name of the node pool.
+   * @param node - datanode id.
+   * @throws SCMException
+   */
+  @Override
+  public void removeNode(final String pool, final DatanodeDetails node)
+      throws SCMException {
+    Preconditions.checkNotNull(pool, "pool name is null");
+    Preconditions.checkNotNull(node, "node is null");
+    lock.writeLock().lock();
+    try {
+      // Remove from the persistent store
+      byte[] kName = node.getProtoBufMessage().toByteArray();
+      byte[] kData = nodePoolStore.get(kName);
+      if (kData == null) {
+        throw new SCMException(String.format("Unable to find node %s from" +
+            " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
+            FAILED_TO_FIND_NODE_IN_POOL);
+      }
+      nodePoolStore.delete(kName);
+
+      // Remove from the in-memory store
+      if (nodePools.containsKey(pool)) {
+        Set<DatanodeDetails> nodePool = nodePools.get(pool);
+        nodePool.remove(node);
+      } else {
+        throw new SCMException(String.format("Unable to find node %s from" +
+            " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
+            FAILED_TO_FIND_NODE_IN_POOL);
+      }
+    } catch (IOException e) {
+      throw new SCMException("Failed to remove node " + node.toString()
+          + " from node pool " + pool, e,
+          SCMException.ResultCodes.IO_EXCEPTION);
+    } finally {
+      lock.writeLock().unlock();
+    }
+  }
+
+  /**
+   * Get all the node pools.
+   * @return all the node pools.
+   */
+  @Override
+  public List<String> getNodePools() {
+    lock.readLock().lock();
+    try {
+      if (!nodePools.isEmpty()) {
+        return nodePools.keySet().stream().collect(Collectors.toList());
+      } else {
+        return EMPTY_NODEPOOL_LIST;
+      }
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  /**
+   * Get all datanodes of a specific node pool.
+   * @param pool - name of the node pool.
+   * @return all datanodes of the specified node pool.
+   */
+  @Override
+  public List<DatanodeDetails> getNodes(final String pool) {
+    Preconditions.checkNotNull(pool, "pool name is null");
+    if (nodePools.containsKey(pool)) {
+      return nodePools.get(pool).stream().collect(Collectors.toList());
+    } else {
+      return EMPTY_NODE_LIST;
+    }
+  }
+
+  /**
+   * Get the node pool name if the node has been added to a node pool.
+   * @param datanodeDetails - datanode ID.
+   * @return node pool name if it has been assigned.
+   * null if the node has not been assigned to any node pool yet.
+   * TODO: Put this in a in-memory map if performance is an issue.
+   */
+  @Override
+  public String getNodePool(final DatanodeDetails datanodeDetails)
+      throws SCMException {
+    Preconditions.checkNotNull(datanodeDetails, "node is null");
+    try {
+      byte[] result = nodePoolStore.get(
+          datanodeDetails.getProtoBufMessage().toByteArray());
+      return result == null ? null : DFSUtil.bytes2String(result);
+    } catch (IOException e) {
+      throw new SCMException("Failed to get node pool for node "
+          + datanodeDetails.toString(), e,
+          SCMException.ResultCodes.IO_EXCEPTION);
+    }
+  }
+
+  /**
+   * Close node pool level db store.
+   * @throws IOException
+   */
+  @Override
+  public void close() throws IOException {
+    nodePoolStore.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 80b5d6e..8c59462 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.container;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -272,6 +273,11 @@ public class MockNodeManager implements NodeManager {
     return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
   }
 
+  @Override
+  public NodePoolManager getNodePoolManager() {
+    return Mockito.mock(NodePoolManager.class);
+  }
+
   /**
    * Used for testing.
    *

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
new file mode 100644
index 0000000..8f412de
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.commons.collections.ListUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms
+    .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms
+    .SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for SCM node pool manager.
+ */
+public class TestSCMNodePoolManager {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestSCMNodePoolManager.class);
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  private final File testDir = PathUtils.getTestDir(
+      TestSCMNodePoolManager.class);
+
+  SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
+      throws IOException {
+    conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+        testDir.getAbsolutePath());
+    conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+        SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+    return new SCMNodePoolManager(conf);
+  }
+
+  /**
+   * Test default node pool.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testDefaultNodePool() throws IOException {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    try {
+      final String defaultPool = "DefaultPool";
+      NodePoolManager npMgr = createNodePoolManager(conf);
+
+      final int nodeCount = 4;
+      final List<DatanodeDetails> nodes = TestUtils
+          .getListOfDatanodeDetails(nodeCount);
+      assertEquals(0, npMgr.getNodePools().size());
+      for (DatanodeDetails node: nodes) {
+        npMgr.addNode(defaultPool, node);
+      }
+      List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+      assertEquals(nodeCount, nodesRetrieved.size());
+      assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+
+      DatanodeDetails nodeRemoved = nodes.remove(2);
+      npMgr.removeNode(defaultPool, nodeRemoved);
+      List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
+      assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
+
+      List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
+      assertEquals(0, nonExistSet.size());
+    } finally {
+      FileUtil.fullyDelete(testDir);
+    }
+  }
+
+
+  /**
+   * Test default node pool reload.
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testDefaultNodePoolReload() throws IOException {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    final String defaultPool = "DefaultPool";
+    final int nodeCount = 4;
+    final List<DatanodeDetails> nodes = TestUtils
+        .getListOfDatanodeDetails(nodeCount);
+
+    try {
+      try {
+        SCMNodePoolManager npMgr = createNodePoolManager(conf);
+        assertEquals(0, npMgr.getNodePools().size());
+        for (DatanodeDetails node : nodes) {
+          npMgr.addNode(defaultPool, node);
+        }
+        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+        assertEquals(nodeCount, nodesRetrieved.size());
+        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+        npMgr.close();
+      } finally {
+        LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
+            " and close.");
+      }
+
+      // try reload with a new NodePoolManager instance
+      try {
+        SCMNodePoolManager npMgr = createNodePoolManager(conf);
+        List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+        assertEquals(nodeCount, nodesRetrieved.size());
+        assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+      } finally {
+        LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
+      }
+    } finally {
+      FileUtil.fullyDelete(testDir);
+    }
+  }
+
+  /**
+   * Compare and verify that two datanode lists are equal.
+   * @param list1 - datanode list 1.
+   * @param list2 - datanode list 2.
+   */
+  private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
+      List<DatanodeDetails> list2) {
+    assertEquals(list1.size(), list2.size());
+    Collections.sort(list1);
+    Collections.sort(list2);
+    assertTrue(ListUtils.isEqualList(list1, list2));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 1a4dcd7..072d821 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.node.CommandQueue;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
 import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -200,6 +201,10 @@ public class ReplicationNodeManagerMock implements NodeManager {
     return null;
   }
 
+  @Override
+  public NodePoolManager getNodePoolManager() {
+    return Mockito.mock(NodePoolManager.class);
+  }
 
   /**
    * Wait for the heartbeat is processed by NodeManager.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
new file mode 100644
index 0000000..ffcd752
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.testutils;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Pool Manager replication mock.
+ */
+public class ReplicationNodePoolManagerMock implements NodePoolManager {
+
+  private final Map<DatanodeDetails, String> nodeMemberShip;
+
+  /**
+   * A node pool manager for testing.
+   */
+  public ReplicationNodePoolManagerMock() {
+    nodeMemberShip = new HashMap<>();
+  }
+
+  /**
+   * Add a node to a node pool.
+   *
+   * @param pool - name of the node pool.
+   * @param node - data node.
+   */
+  @Override
+  public void addNode(String pool, DatanodeDetails node) {
+    nodeMemberShip.put(node, pool);
+  }
+
+  /**
+   * Remove a node from a node pool.
+   *
+   * @param pool - name of the node pool.
+   * @param node - data node.
+   * @throws SCMException
+   */
+  @Override
+  public void removeNode(String pool, DatanodeDetails node)
+      throws SCMException {
+    nodeMemberShip.remove(node);
+
+  }
+
+  /**
+   * Get a list of known node pools.
+   *
+   * @return a list of known node pool names or an empty list if not node pool
+   * is defined.
+   */
+  @Override
+  public List<String> getNodePools() {
+    Set<String> poolSet = new HashSet<>();
+    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
+      poolSet.add(entry.getValue());
+    }
+    return new ArrayList<>(poolSet);
+
+  }
+
+  /**
+   * Get all nodes of a node pool given the name of the node pool.
+   *
+   * @param pool - name of the node pool.
+   * @return a list of datanode ids or an empty list if the node pool was not
+   * found.
+   */
+  @Override
+  public List<DatanodeDetails> getNodes(String pool) {
+    Set<DatanodeDetails> datanodeSet = new HashSet<>();
+    for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
+      if (entry.getValue().equals(pool)) {
+        datanodeSet.add(entry.getKey());
+      }
+    }
+    return new ArrayList<>(datanodeSet);
+  }
+
+  /**
+   * Get the node pool name if the node has been added to a node pool.
+   *
+   * @param datanodeDetails DatanodeDetails.
+   * @return node pool name if it has been assigned. null if the node has not
+   * been assigned to any node pool yet.
+   */
+  @Override
+  public String getNodePool(DatanodeDetails datanodeDetails) {
+    return nodeMemberShip.get(datanodeDetails);
+  }
+
+  /**
+   * Closes this stream and releases any system resources associated
+   * with it. If the stream is already closed then invoking this
+   * method has no effect.
+   * <p>
+   * <p> As noted in {@link AutoCloseable#close()}, cases where the
+   * close may fail require careful attention. It is strongly advised
+   * to relinquish the underlying resources and to internally
+   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+   * the {@code IOException}.
+   *
+   * @throws IOException if an I/O error occurs
+   */
+  @Override
+  public void close() throws IOException {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index b4ed2b1..4d70af8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,9 +51,12 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.UUID;
 
+import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
 import static org.apache.hadoop.ozone.OzoneConsts.KB;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * This class tests the CLI that transforms container into SQLite DB files.
@@ -174,6 +177,34 @@ public class TestContainerSQLCli {
   }
 
   @Test
+  public void testConvertNodepoolDB() throws Exception {
+    String dbOutPath = GenericTestUtils.getTempPath(
+        UUID.randomUUID() + "/out_sql.db");
+    String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+    String dbPath = dbRootPath + "/" + NODEPOOL_DB;
+    String[] args = {"-p", dbPath, "-o", dbOutPath};
+
+    cli.run(args);
+
+    // verify the sqlite db
+    HashMap<String, String> expectedPool = new HashMap<>();
+    for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
+      expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
+    }
+    Connection conn = connectDB(dbOutPath);
+    String sql = "SELECT * FROM nodePool";
+    ResultSet rs = executeQuery(conn, sql);
+    while(rs.next()) {
+      String datanodeUUID = rs.getString("datanodeUUID");
+      String poolName = rs.getString("poolName");
+      assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
+    }
+    assertEquals(0, expectedPool.size());
+
+    Files.delete(Paths.get(dbOutPath));
+  }
+
+  @Test
   public void testConvertContainerDB() throws Exception {
     String dbOutPath = GenericTestUtils.getTempPath(
         UUID.randomUUID() + "/out_sql.db");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[20/50] [abbrv] hadoop git commit: YARN-8108. Added option to disable loading existing filters to prevent security filter from initialize twice. Contributed by Sunil Govindan

Posted by gi...@apache.org.
YARN-8108.  Added option to disable loading existing filters to prevent
            security filter from initialize twice.
            Contributed by Sunil Govindan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b69ba0f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b69ba0f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b69ba0f3

Branch: refs/heads/HADOOP-15461
Commit: b69ba0f3307a90500aeb0c5db9e582fcda60b501
Parents: 3e58633
Author: Eric Yang <ey...@apache.org>
Authored: Tue Jun 26 17:34:57 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Jun 26 17:34:57 2018 -0400

----------------------------------------------------------------------
 .../java/org/apache/hadoop/yarn/webapp/WebApps.java   | 14 +++++++++++---
 .../yarn/server/resourcemanager/ResourceManager.java  |  2 +-
 2 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69ba0f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 0d045f3..0e9f0a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -83,6 +83,7 @@ public class WebApps {
       public String name;
       public String spec;
       public Map<String, String> params;
+      public boolean loadExistingFilters = true;
     }
     
     final String name;
@@ -151,12 +152,13 @@ public class WebApps {
 
     public Builder<T> withServlet(String name, String pathSpec,
         Class<? extends HttpServlet> servlet,
-        Map<String, String> params) {
+        Map<String, String> params,boolean loadExistingFilters) {
       ServletStruct struct = new ServletStruct();
       struct.clazz = servlet;
       struct.name = name;
       struct.spec = pathSpec;
       struct.params = params;
+      struct.loadExistingFilters = loadExistingFilters;
       servlets.add(struct);
       return this;
     }
@@ -256,9 +258,15 @@ public class WebApps {
           pathList.add("/" + wsName + "/*");
         }
       }
+
       for (ServletStruct s : servlets) {
         if (!pathList.contains(s.spec)) {
-          pathList.add(s.spec);
+          // The servlet told us to not load-existing filters, but we still want
+          // to add the default authentication filter always, so add it to the
+          // pathList
+          if (!s.loadExistingFilters) {
+            pathList.add(s.spec);
+          }
         }
       }
       if (conf == null) {
@@ -333,7 +341,7 @@ public class WebApps {
         HttpServer2 server = builder.build();
 
         for(ServletStruct struct: servlets) {
-          if (struct.params != null) {
+          if (!struct.loadExistingFilters) {
             server.addInternalServlet(struct.name, struct.spec,
                 struct.clazz, struct.params);
           } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69ba0f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index c533111..0b7e87c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1111,7 +1111,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
                 "ws")
             .with(conf)
             .withServlet("API-Service", "/app/*",
-                ServletContainer.class, params)
+                ServletContainer.class, params, false)
             .withHttpSpnegoPrincipalKey(
                 YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
             .withHttpSpnegoKeytabKey(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org