You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by gi...@apache.org on 2018/06/29 20:13:52 UTC
[01/50] [abbrv] hadoop git commit: YARN-8444: NodeResourceMonitor
crashes on bad swapFree value. Contributed by Jim Brennan. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-15461 1c0b63280 -> 75f23cbf6 (forced update)
YARN-8444: NodeResourceMonitor crashes on bad swapFree value. Contributed by Jim Brennan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64321286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64321286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64321286
Branch: refs/heads/HADOOP-15461
Commit: 6432128622d64f3f9dd638b9c254c77cdf5408aa
Parents: 30728ac
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 22 17:15:29 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 22 17:15:29 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/util/SysInfoLinux.java | 19 ++++++-
.../apache/hadoop/util/TestSysInfoLinux.java | 60 ++++++++++++++++++++
2 files changed, 77 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
index 7fd1990..2c2aca3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java
@@ -217,6 +217,21 @@ public class SysInfoLinux extends SysInfo {
}
/**
+ *
+ * Wrapper for Long.parseLong() that returns zero if the value is
+ * invalid. Under some circumstances, swapFree in /proc/meminfo can
+ * go negative, reported as a very large decimal value.
+ */
+ private long safeParseLong(String strVal) {
+ long parsedVal;
+ try {
+ parsedVal = Long.parseLong(strVal);
+ } catch (NumberFormatException nfe) {
+ parsedVal = 0;
+ }
+ return parsedVal;
+ }
+ /**
* Read /proc/meminfo, parse and compute memory information.
* @param readAgain if false, read only on the first time
*/
@@ -252,9 +267,9 @@ public class SysInfoLinux extends SysInfo {
} else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
swapSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(MEMFREE_STRING)) {
- ramSizeFree = Long.parseLong(mat.group(2));
+ ramSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(SWAPFREE_STRING)) {
- swapSizeFree = Long.parseLong(mat.group(2));
+ swapSizeFree = safeParseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVE_STRING)) {
inactiveSize = Long.parseLong(mat.group(2));
} else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64321286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
index a646a41..0ae5d3c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java
@@ -161,6 +161,36 @@ public class TestSysInfoLinux {
"DirectMap2M: 2027520 kB\n" +
"DirectMap1G: 132120576 kB\n";
+ static final String MEMINFO_FORMAT3 =
+ "MemTotal: %d kB\n" +
+ "MemFree: %s kB\n" +
+ "Buffers: 138244 kB\n" +
+ "Cached: 947780 kB\n" +
+ "SwapCached: 142880 kB\n" +
+ "Active: 3229888 kB\n" +
+ "Inactive: %d kB\n" +
+ "SwapTotal: %d kB\n" +
+ "SwapFree: %s kB\n" +
+ "Dirty: 122012 kB\n" +
+ "Writeback: 0 kB\n" +
+ "AnonPages: 2710792 kB\n" +
+ "Mapped: 24740 kB\n" +
+ "Slab: 132528 kB\n" +
+ "SReclaimable: 105096 kB\n" +
+ "SUnreclaim: 27432 kB\n" +
+ "PageTables: 11448 kB\n" +
+ "NFS_Unstable: 0 kB\n" +
+ "Bounce: 0 kB\n" +
+ "CommitLimit: 4125904 kB\n" +
+ "Committed_AS: 4143556 kB\n" +
+ "VmallocTotal: 34359738367 kB\n" +
+ "VmallocUsed: 1632 kB\n" +
+ "VmallocChunk: 34359736375 kB\n" +
+ "HugePages_Total: %d\n" +
+ "HugePages_Free: 0\n" +
+ "HugePages_Rsvd: 0\n" +
+ "Hugepagesize: 2048 kB";
+
static final String CPUINFO_FORMAT =
"processor : %s\n" +
"vendor_id : AuthenticAMD\n" +
@@ -384,6 +414,36 @@ public class TestSysInfoLinux {
(nrHugePages * 2048) + swapTotal));
}
+ /**
+ * Test parsing /proc/meminfo
+ * @throws IOException
+ */
+ @Test
+ public void parsingProcMemFileWithBadValues() throws IOException {
+ long memTotal = 4058864L;
+ long memFree = 0L; // bad value should return 0
+ long inactive = 567732L;
+ long swapTotal = 2096472L;
+ long swapFree = 0L; // bad value should return 0
+ int nrHugePages = 10;
+ String badFreeValue = "18446744073709551596";
+ File tempFile = new File(FAKE_MEMFILE);
+ tempFile.deleteOnExit();
+ FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+ fWriter.write(String.format(MEMINFO_FORMAT3,
+ memTotal, badFreeValue, inactive, swapTotal, badFreeValue, nrHugePages));
+
+ fWriter.close();
+ assertEquals(plugin.getAvailablePhysicalMemorySize(),
+ 1024L * (memFree + inactive));
+ assertEquals(plugin.getAvailableVirtualMemorySize(),
+ 1024L * (memFree + inactive + swapFree));
+ assertEquals(plugin.getPhysicalMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048)));
+ assertEquals(plugin.getVirtualMemorySize(),
+ 1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+ }
+
@Test
public void testCoreCounts() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-8464. Async scheduling thread
could be interrupted when there are no NodeManagers in cluster. (Sunil G via
wangda)
Posted by gi...@apache.org.
YARN-8464. Async scheduling thread could be interrupted when there are no NodeManagers in cluster. (Sunil G via wangda)
Change-Id: I4f5f856373378685713e77752ba6cf0988a66065
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bedc4fe0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bedc4fe0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bedc4fe0
Branch: refs/heads/HADOOP-15461
Commit: bedc4fe0799cf3b161100acc521fc62a97793427
Parents: ada8f63
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jun 26 19:27:17 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jun 26 19:27:17 2018 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bedc4fe0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 50ab70d..54bbf24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -521,7 +521,14 @@ public class CapacityScheduler extends
// First randomize the start point
int current = 0;
Collection<FiCaSchedulerNode> nodes = cs.nodeTracker.getAllNodes();
- int start = random.nextInt(nodes.size());
+
+ // If nodes size is 0 (when there are no node managers registered,
+ // we can return from here itself.
+ int nodeSize = nodes.size();
+ if(nodeSize == 0) {
+ return;
+ }
+ int start = random.nextInt(nodeSize);
// To avoid too verbose DEBUG logging, only print debug log once for
// every 10 secs.
@@ -574,6 +581,7 @@ public class CapacityScheduler extends
@Override
public void run() {
+ int debuggingLogCounter = 0;
while (!Thread.currentThread().isInterrupted()) {
try {
if (!runSchedules.get()) {
@@ -585,6 +593,14 @@ public class CapacityScheduler extends
Thread.sleep(1);
} else{
schedule(cs);
+ if(LOG.isDebugEnabled()) {
+ // Adding a debug log here to ensure that the thread is alive
+ // and running fine.
+ if (debuggingLogCounter++ > 10000) {
+ debuggingLogCounter = 0;
+ LOG.debug("AsyncScheduleThread[" + getName() + "] is running!");
+ }
+ }
}
}
} catch (InterruptedException ie) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-8469. [UI2] URL needs to be
trimmed to handle index.html redirection while accessing via knox.
Contributed by Sunil Govindan.
Posted by gi...@apache.org.
YARN-8469. [UI2] URL needs to be trimmed to handle index.html redirection while accessing via knox. Contributed by Sunil Govindan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a820738e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a820738e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a820738e
Branch: refs/heads/HADOOP-15461
Commit: a820738e347854aed7cdaa7758d9c0830df624e3
Parents: 469b29c
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Fri Jun 29 10:21:25 2018 -0700
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Fri Jun 29 10:21:25 2018 -0700
----------------------------------------------------------------------
.../src/main/webapp/app/initializers/loader.js | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a820738e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
index 6642869..96c8fcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/initializers/loader.js
@@ -205,5 +205,13 @@ export default {
const skipTrailingSlash = function(path) {
path = path.replace('ui2/', '');
- return path.replace(/\/$/, '');
+ path = path.replace(/\/$/, '');
+ console.log('base url:' + path)
+ if(path.includes("redirect")) {
+ var to = path.lastIndexOf('/');
+ to = to == -1 ? path.length : to + 1;
+ path = path.substring(0, to);
+ console.log('base url after redirect:' + path)
+ }
+ return path;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HADOOP-15465. Deprecate
WinUtils#Symlinks by using native java code. Contributed by Giovanni Matteo
Fumarola.
Posted by gi...@apache.org.
HADOOP-15465. Deprecate WinUtils#Symlinks by using native java code. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/32372eac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/32372eac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/32372eac
Branch: refs/heads/HADOOP-15461
Commit: 32372eacbfc13451e4be6254602b38f632582b0c
Parents: cdb0844
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Jun 7 17:02:01 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 13:08:57 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 60 ++++++++------------
.../apache/hadoop/fs/RawLocalFileSystem.java | 2 -
.../main/java/org/apache/hadoop/util/Shell.java | 9 ++-
.../hadoop/yarn/server/MiniYARNCluster.java | 13 ++---
4 files changed, 37 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index df89598..61cb8d2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -34,8 +34,10 @@ import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.file.AccessDeniedException;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
@@ -1028,17 +1030,15 @@ public class FileUtil {
}
/**
- * Create a soft link between a src and destination
- * only on a local disk. HDFS does not support this.
- * On Windows, when symlink creation fails due to security
- * setting, we will log a warning. The return code in this
- * case is 2.
+ * Create a soft link between a src and destination only on a local disk. On
+ * Windows, when symlink creation fails due to security setting, we will log a
+ * warning. The return code in this case is 2.
*
* @param target the target for symlink
* @param linkname the symlink
* @return 0 on success
*/
- public static int symLink(String target, String linkname) throws IOException{
+ public static int symLink(String target, String linkname) throws IOException {
if (target == null || linkname == null) {
LOG.warn("Can not create a symLink with a target = " + target
@@ -1053,44 +1053,32 @@ public class FileUtil {
File linkFile = new File(
Path.getPathWithoutSchemeAndAuthority(new Path(linkname)).toString());
- String[] cmd = Shell.getSymlinkCommand(
- targetFile.toString(),
- linkFile.toString());
-
- ShellCommandExecutor shExec;
try {
- if (Shell.WINDOWS &&
- linkFile.getParentFile() != null &&
- !new Path(target).isAbsolute()) {
- // Relative links on Windows must be resolvable at the time of
- // creation. To ensure this we run the shell command in the directory
- // of the link.
- //
- shExec = new ShellCommandExecutor(cmd, linkFile.getParentFile());
- } else {
- shExec = new ShellCommandExecutor(cmd);
- }
- shExec.execute();
- } catch (Shell.ExitCodeException ec) {
- int returnVal = ec.getExitCode();
- if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) {
- LOG.warn("Fail to create symbolic links on Windows. "
- + "The default security settings in Windows disallow non-elevated "
- + "administrators and all non-administrators from creating symbolic links. "
- + "This behavior can be changed in the Local Security Policy management console");
- } else if (returnVal != 0) {
- LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed "
- + returnVal + " with: " + ec.getMessage());
- }
- return returnVal;
+ Files.createSymbolicLink(Paths.get(linkFile.toString()),
+ Paths.get(targetFile.toString()));
+ } catch (SecurityException e3) {
+ LOG.warn("Fail to create symbolic links on Windows. "
+ + "The default security settings in Windows disallow non-elevated "
+ + "administrators and all non-administrators from creating symbolic"
+ + " links. This behavior can be changed in the Local Security Policy"
+ + " management console");
+ return SYMLINK_NO_PRIVILEGE;
+
+ } catch (FileAlreadyExistsException | UnsupportedOperationException e) {
+ LOG.warn("Fail to create symbolic links. ErrorMessage = "
+ + e.getLocalizedMessage());
+ return 1;
+
} catch (IOException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Error while create symlink " + linkname + " to " + target
+ "." + " Exception: " + StringUtils.stringifyException(e));
}
throw e;
+
}
- return shExec.getExitCode();
+
+ return 0;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index bd003ae..8333366 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -925,7 +925,6 @@ public class RawLocalFileSystem extends FileSystem {
return true;
}
- @SuppressWarnings("deprecation")
@Override
public void createSymlink(Path target, Path link, boolean createParent)
throws IOException {
@@ -941,7 +940,6 @@ public class RawLocalFileSystem extends FileSystem {
mkdirs(link.getParent());
}
- // NB: Use createSymbolicLink in java.nio.file.Path once available
int result = FileUtil.symLink(target.toString(),
makeAbsolute(link).toString());
if (result != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 46a0fcc..e902af0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -37,7 +37,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -297,7 +296,13 @@ public abstract class Shell {
: new String[] { "chown", owner };
}
- /** Return a command to create symbolic links. */
+ /**
+ * Return a command to create symbolic links.
+ *
+ * Deprecated and likely to be deleted in the near future. Please use
+ * FileUtil.symlink().
+ */
+ @Deprecated
public static String[] getSymlinkCommand(String target, String link) {
return WINDOWS ?
new String[] { getWinUtilsPath(), "symlink", link, target }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/32372eac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 0395138..01ac02c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -42,7 +43,6 @@ import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.HAUtil;
@@ -202,14 +202,13 @@ public class MiniYARNCluster extends CompositeService {
// Guarantee target exists before creating symlink.
targetWorkDir.mkdirs();
- ShellCommandExecutor shexec = new ShellCommandExecutor(
- Shell.getSymlinkCommand(targetPath, linkPath));
try {
- shexec.execute();
+ FileUtil.symLink(targetPath, linkPath);
} catch (IOException e) {
- throw new YarnRuntimeException(String.format(
- "failed to create symlink from %s to %s, shell output: %s", linkPath,
- targetPath, shexec.getOutput()), e);
+ throw new YarnRuntimeException(
+ String.format("failed to create symlink from %s to %s.",
+ linkPath, targetPath),
+ e);
}
this.testWorkDir = link;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager
and node pool handling from SCM. Contributed by Elek Marton
Posted by gi...@apache.org.
HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaf03cc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaf03cc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaf03cc4
Branch: refs/heads/HADOOP-15461
Commit: aaf03cc459a34af284f9735453aefd4ddb430d67
Parents: fbaff36
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 12:39:15 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 12:39:15 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 --
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 -
.../common/src/main/resources/ozone-default.xml | 47 ---
.../container/replication/ReplicationQueue.java | 78 +++++
.../replication/ReplicationReqMsg.java | 107 ++++++
.../container/replication/package-info.java | 23 ++
.../replication/TestReplicationQueue.java | 134 ++++++++
.../container/replication/package-info.java | 23 ++
.../hdds/scm/container/ContainerMapping.java | 10 +-
.../replication/ContainerSupervisor.java | 340 -------------------
.../container/replication/InProgressPool.java | 255 --------------
.../scm/container/replication/PeriodicPool.java | 119 -------
.../scm/container/replication/package-info.java | 23 --
.../hadoop/hdds/scm/node/NodeManager.java | 6 -
.../hadoop/hdds/scm/node/NodePoolManager.java | 71 ----
.../hadoop/hdds/scm/node/SCMNodeManager.java | 23 --
.../hdds/scm/node/SCMNodePoolManager.java | 269 ---------------
.../hdds/scm/container/MockNodeManager.java | 6 -
.../hdds/scm/node/TestSCMNodePoolManager.java | 160 ---------
.../testutils/ReplicationNodeManagerMock.java | 5 -
.../ReplicationNodePoolManagerMock.java | 133 --------
.../hadoop/ozone/scm/TestContainerSQLCli.java | 31 --
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 74 ----
23 files changed, 368 insertions(+), 1596 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 85407e6..df6fbf0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,32 +243,6 @@ public final class ScmConfigKeys {
public static final String
OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
- /**
- * Don't start processing a pool if we have not had a minimum number of
- * seconds from the last processing.
- */
- public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
- "ozone.scm.container.report.processing.interval";
- public static final String
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
- /**
- * This determines the total number of pools to be processed in parallel.
- */
- public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
- "ozone.scm.max.nodepool.processing.threads";
- public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
- /**
- * These 2 settings control the number of threads in executor pool and time
- * outs for thw container reports from all nodes.
- */
- public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
- "ozone.scm.max.container.report.threads";
- public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
- "ozone.scm.container.reports.wait.timeout";
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
- "5m";
public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
"ozone.scm.block.deletion.max.retry";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index c40dc8e..08a5ffd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,7 +91,6 @@ public final class OzoneConsts {
public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String BLOCK_DB = "block.db";
- public static final String NODEPOOL_DB = "nodepool.db";
public static final String OPEN_CONTAINERS_DB = "openContainers.db";
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String KSM_DB_NAME = "ksm.db";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7a91610..25365c8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,25 +572,6 @@
</description>
</property>
<property>
- <name>ozone.scm.container.report.processing.interval</name>
- <value>60s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Time interval for scm to process container reports
- for a node pool. Scm handles node pool reports in a cyclic clock
- manner, it fetches pools periodically with this time interval.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.reports.wait.timeout</name>
- <value>300s</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>Maximum time to wait in seconds for processing all container
- reports from
- a node pool. It determines the timeout for a
- node pool report.
- </description>
- </property>
- <property>
<name>ozone.scm.container.size.gb</name>
<value>5</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -793,17 +774,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.container.report.threads</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Maximum number of threads to process container reports in scm.
- Each container report from a data node is processed by scm in a worker
- thread, fetched from a thread pool. This property is used to control the
- maximum size of the thread pool.
- </description>
- </property>
- <property>
<name>ozone.scm.max.hb.count.to.process</name>
<value>5000</value>
<tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -815,14 +785,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- Number of node pools to process in parallel.
- </description>
- </property>
- <property>
<name>ozone.scm.names</name>
<value/>
<tag>OZONE</tag>
@@ -844,15 +806,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, SCM</tag>
- <description>
- Controls the number of node pools that can be processed in parallel by
- Container Supervisor.
- </description>
- </property>
- <property>
<name>ozone.trace.enabled</name>
<value>false</value>
<tag>OZONE, DEBUG</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
new file mode 100644
index 0000000..b83ecf1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+
+/**
+ * Priority queue to handle under-replicated and over replicated containers
+ * in ozone. ReplicationManager will consume these messages and decide
+ * accordingly.
+ */
+public class ReplicationQueue {
+
+ private final Queue<ReplicationReqMsg> queue;
+
+ ReplicationQueue() {
+ queue = new PriorityQueue<>();
+ }
+
+ public synchronized boolean add(ReplicationReqMsg repObj) {
+ if (this.queue.contains(repObj)) {
+ // Remove the earlier message and insert this one
+ this.queue.remove(repObj);
+ return this.queue.add(repObj);
+ } else {
+ return this.queue.add(repObj);
+ }
+ }
+
+ public synchronized boolean remove(ReplicationReqMsg repObj) {
+ return queue.remove(repObj);
+ }
+
+ /**
+ * Retrieves, but does not remove, the head of this queue,
+ * or returns {@code null} if this queue is empty.
+ *
+ * @return the head of this queue, or {@code null} if this queue is empty
+ */
+ public synchronized ReplicationReqMsg peek() {
+ return queue.peek();
+ }
+
+ /**
+ * Retrieves and removes the head of this queue,
+ * or returns {@code null} if this queue is empty.
+ *
+ * @return the head of this queue, or {@code null} if this queue is empty
+ */
+ public synchronized ReplicationReqMsg poll() {
+ return queue.poll();
+ }
+
+ public synchronized boolean removeAll(List<ReplicationReqMsg> repObjs) {
+ return queue.removeAll(repObjs);
+ }
+
+ public int size() {
+ return queue.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
new file mode 100644
index 0000000..8d26fc3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.commons.lang3.math.NumberUtils;
+
+/**
+ * Wrapper class for hdds replication queue. Implements its natural
+ * ordering for priority queue.
+ */
+public class ReplicationReqMsg implements Comparable<ReplicationReqMsg>,
+ Serializable {
+ private final long containerId;
+ private final short replicationCount;
+ private final short expecReplicationCount;
+ private final long timestamp;
+
+ public ReplicationReqMsg(long containerId, short replicationCount,
+ long timestamp, short expecReplicationCount) {
+ this.containerId = containerId;
+ this.replicationCount = replicationCount;
+ this.timestamp = timestamp;
+ this.expecReplicationCount = expecReplicationCount;
+ }
+
+ /**
+ * Compares this object with the specified object for order. Returns a
+ * negative integer, zero, or a positive integer as this object is less
+ * than, equal to, or greater than the specified object.
+ * @param o the object to be compared.
+ * @return a negative integer, zero, or a positive integer as this object
+ * is less than, equal to, or greater than the specified object.
+ * @throws NullPointerException if the specified object is null
+ * @throws ClassCastException if the specified object's type prevents it
+ * from being compared to this object.
+ */
+ @Override
+ public int compareTo(ReplicationReqMsg o) {
+ if (this == o) {
+ return 0;
+ }
+ if (o == null) {
+ return 1;
+ }
+ int retVal = NumberUtils
+ .compare(getReplicationCount() - getExpecReplicationCount(),
+ o.getReplicationCount() - o.getExpecReplicationCount());
+ if (retVal != 0) {
+ return retVal;
+ }
+ return NumberUtils.compare(getTimestamp(), o.getTimestamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(91, 1011)
+ .append(getContainerId())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationReqMsg that = (ReplicationReqMsg) o;
+ return new EqualsBuilder().append(getContainerId(), that.getContainerId())
+ .isEquals();
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ public short getReplicationCount() {
+ return replicationCount;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public short getExpecReplicationCount() {
+ return expecReplicationCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..7f335e3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Ozone Container replicaton related classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
new file mode 100644
index 0000000..39c61d3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.Random;
+import java.util.UUID;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for ReplicationQueue.
+ */
+public class TestReplicationQueue {
+
+ private ReplicationQueue replicationQueue;
+ private Random random;
+
+ @Before
+ public void setUp() {
+ replicationQueue = new ReplicationQueue();
+ random = new Random();
+ }
+
+ @Test
+ public void testDuplicateAddOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationReqMsg obj1, obj2, obj3;
+ long time = Time.monotonicNow();
+ obj1 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
+ obj2 = new ReplicationReqMsg(contId, (short) 2, time + 1, (short) 3);
+ obj3 = new ReplicationReqMsg(contId, (short) 1, time+2, (short) 3);
+
+ replicationQueue.add(obj1);
+ replicationQueue.add(obj2);
+ replicationQueue.add(obj3);
+ Assert.assertEquals("Should add only 1 msg as second one is duplicate",
+ 1, replicationQueue.size());
+ ReplicationReqMsg temp = replicationQueue.poll();
+ Assert.assertEquals(temp, obj3);
+ }
+
+ @Test
+ public void testPollOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationReqMsg msg1, msg2, msg3, msg4, msg5;
+ msg1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
+ (short) 3);
+ long time = Time.monotonicNow();
+ msg2 = new ReplicationReqMsg(contId + 1, (short) 4, time, (short) 3);
+ msg3 = new ReplicationReqMsg(contId + 2, (short) 0, time, (short) 3);
+ msg4 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
+ // Replication message for same container but different nodeId
+ msg5 = new ReplicationReqMsg(contId + 1, (short) 2, time, (short) 3);
+
+ replicationQueue.add(msg1);
+ replicationQueue.add(msg2);
+ replicationQueue.add(msg3);
+ replicationQueue.add(msg4);
+ replicationQueue.add(msg5);
+ Assert.assertEquals("Should have 3 objects",
+ 3, replicationQueue.size());
+
+ // Since Priority queue orders messages according to replication count,
+ // message with lowest replication should be first
+ ReplicationReqMsg temp;
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 2 objects",
+ 2, replicationQueue.size());
+ Assert.assertEquals(temp, msg3);
+
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 1 objects",
+ 1, replicationQueue.size());
+ Assert.assertEquals(temp, msg5);
+
+ // Message 2 should be ordered before message 5 as both have same replication
+ // number but message 2 has earlier timestamp.
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 0 objects",
+ replicationQueue.size(), 0);
+ Assert.assertEquals(temp, msg4);
+ }
+
+ @Test
+ public void testRemoveOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationReqMsg obj1, obj2, obj3;
+ obj1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
+ (short) 3);
+ obj2 = new ReplicationReqMsg(contId + 1, (short) 2, Time.monotonicNow(),
+ (short) 3);
+ obj3 = new ReplicationReqMsg(contId + 2, (short) 3, Time.monotonicNow(),
+ (short) 3);
+
+ replicationQueue.add(obj1);
+ replicationQueue.add(obj2);
+ replicationQueue.add(obj3);
+ Assert.assertEquals("Should have 3 objects",
+ 3, replicationQueue.size());
+
+ replicationQueue.remove(obj3);
+ Assert.assertEquals("Should have 2 objects",
+ 2, replicationQueue.size());
+
+ replicationQueue.remove(obj2);
+ Assert.assertEquals("Should have 1 objects",
+ 1, replicationQueue.size());
+
+ replicationQueue.remove(obj1);
+ Assert.assertEquals("Should have 0 objects",
+ 0, replicationQueue.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5b1fd0f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * SCM Testing and Mocking Utils.
+ */
+package org.apache.hadoop.ozone.container.replication;
+// Test classes for Replication functionality.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index b563e90..9fd30f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -80,7 +79,6 @@ public class ContainerMapping implements Mapping {
private final PipelineSelector pipelineSelector;
private final ContainerStateManager containerStateManager;
private final LeaseManager<ContainerInfo> containerLeaseManager;
- private final ContainerSupervisor containerSupervisor;
private final float containerCloseThreshold;
private final ContainerCloser closer;
private final long size;
@@ -127,9 +125,7 @@ public class ContainerMapping implements Mapping {
OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
this.containerStateManager =
new ContainerStateManager(conf, this);
- this.containerSupervisor =
- new ContainerSupervisor(conf, nodeManager,
- nodeManager.getNodePoolManager());
+
this.containerCloseThreshold = conf.getFloat(
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -407,8 +403,8 @@ public class ContainerMapping implements Mapping {
throws IOException {
List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
containerInfos = reports.getReportsList();
- containerSupervisor.handleContainerReport(datanodeDetails, reports);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
containerInfos) {
byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
lock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index 5bd0574..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
- .sleepUninterruptibly;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
- public static final Logger LOG =
- LoggerFactory.getLogger(ContainerSupervisor.class);
-
- private final NodePoolManager poolManager;
- private final HashSet<String> poolNames;
- private final PriorityQueue<PeriodicPool> poolQueue;
- private final NodeManager nodeManager;
- private final long containerProcessingLag;
- private final AtomicBoolean runnable;
- private final ExecutorService executorService;
- private final long maxPoolWait;
- private long poolProcessCount;
- private final List<InProgressPool> inProgressPoolList;
- private final AtomicInteger threadFaultCount;
- private final int inProgressPoolMaxCount;
-
- private final ReadWriteLock inProgressPoolListLock;
-
- /**
- * Returns the number of times we have processed pools.
- * @return long
- */
- public long getPoolProcessCount() {
- return poolProcessCount;
- }
-
-
- /**
- * Constructs a class that computes Replication Levels.
- *
- * @param conf - OzoneConfiguration
- * @param nodeManager - Node Manager
- * @param poolManager - Pool Manager
- */
- public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
- NodePoolManager poolManager) {
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(nodeManager);
- this.containerProcessingLag =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
- TimeUnit.SECONDS
- ) * 1000;
- int maxContainerReportThreads =
- conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
- OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
- );
- this.maxPoolWait =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
- OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.inProgressPoolMaxCount = conf.getInt(
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
- this.poolManager = poolManager;
- this.nodeManager = nodeManager;
- this.poolNames = new HashSet<>();
- this.poolQueue = new PriorityQueue<>();
- this.runnable = new AtomicBoolean(true);
- this.threadFaultCount = new AtomicInteger(0);
- this.executorService = newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Container Reports Processing Thread - %d")
- .build(), maxContainerReportThreads);
- this.inProgressPoolList = new LinkedList<>();
- this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
- initPoolProcessThread();
- }
-
- private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
- int maxThreads) {
- return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(), threadFactory);
- }
-
- /**
- * Returns the number of pools that are under process right now.
- * @return int - Number of pools that are in process.
- */
- public int getInProgressPoolCount() {
- return inProgressPoolList.size();
- }
-
- /**
- * Exits the background thread.
- */
- public void setExit() {
- this.runnable.set(false);
- }
-
- /**
- * Adds or removes pools from names that we need to process.
- *
- * There are two different cases that we need to process.
- * The case where some pools are being added and some times we have to
- * handle cases where pools are removed.
- */
- private void refreshPools() {
- List<String> pools = this.poolManager.getNodePools();
- if (pools != null) {
-
- HashSet<String> removedPools =
- computePoolDifference(this.poolNames, new HashSet<>(pools));
-
- HashSet<String> addedPools =
- computePoolDifference(new HashSet<>(pools), this.poolNames);
- // TODO: Support remove pool API in pool manager so that this code
- // path can be tested. This never happens in the current code base.
- for (String poolName : removedPools) {
- for (PeriodicPool periodicPool : poolQueue) {
- if (periodicPool.getPoolName().compareTo(poolName) == 0) {
- poolQueue.remove(periodicPool);
- }
- }
- }
- // Remove the pool names that we have in the list.
- this.poolNames.removeAll(removedPools);
-
- for (String poolName : addedPools) {
- poolQueue.add(new PeriodicPool(poolName));
- }
-
- // Add to the pool names we are tracking.
- poolNames.addAll(addedPools);
- }
-
- }
-
- /**
- * Handle the case where pools are added.
- *
- * @param newPools - New Pools list
- * @param oldPool - oldPool List.
- */
- private HashSet<String> computePoolDifference(HashSet<String> newPools,
- Set<String> oldPool) {
- Preconditions.checkNotNull(newPools);
- Preconditions.checkNotNull(oldPool);
- HashSet<String> newSet = new HashSet<>(newPools);
- newSet.removeAll(oldPool);
- return newSet;
- }
-
- private void initPoolProcessThread() {
-
- /*
- * Task that runs to check if we need to start a pool processing job.
- * if so we create a pool reconciliation job and find out of all the
- * expected containers are on the nodes.
- */
- Runnable processPools = () -> {
- while (runnable.get()) {
- // Make sure that we don't have any new pools.
- refreshPools();
- while (inProgressPoolList.size() < inProgressPoolMaxCount) {
- PeriodicPool pool = poolQueue.poll();
- if (pool != null) {
- if (pool.getLastProcessedTime() + this.containerProcessingLag >
- Time.monotonicNow()) {
- LOG.debug("Not within the time window for processing: {}",
- pool.getPoolName());
- // we might over sleep here, not a big deal.
- sleepUninterruptibly(this.containerProcessingLag,
- TimeUnit.MILLISECONDS);
- }
- LOG.debug("Adding pool {} to container processing queue",
- pool.getPoolName());
- InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
- pool, this.nodeManager, this.poolManager, this.executorService);
- inProgressPool.startReconciliation();
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.add(inProgressPool);
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- poolProcessCount++;
- } else {
- break;
- }
- }
- sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
- inProgressPoolListLock.readLock().lock();
- try {
- for (InProgressPool inProgressPool : inProgressPoolList) {
- inProgressPool.finalizeReconciliation();
- poolQueue.add(inProgressPool.getPool());
- }
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.clear();
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- }
- };
-
- // We will have only one thread for pool processing.
- Thread poolProcessThread = new Thread(processPools);
- poolProcessThread.setDaemon(true);
- poolProcessThread.setName("Pool replica thread");
- poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
- // Let us just restart this thread after logging a critical error.
- // if this thread is not running we cannot handle commands from SCM.
- LOG.error("Critical Error : Pool replica thread encountered an " +
- "error. Thread: {} Error Count : {}", t.toString(), e,
- threadFaultCount.incrementAndGet());
- poolProcessThread.start();
- // TODO : Add a config to restrict how many times we will restart this
- // thread in a single session.
- });
- poolProcessThread.start();
- }
-
- /**
- * Adds a container report to appropriate inProgress Pool.
- * @param containerReport -- Container report for a specific container from
- * a datanode.
- */
- public void handleContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto containerReport) {
- inProgressPoolListLock.readLock().lock();
- try {
- String poolName = poolManager.getNodePool(datanodeDetails);
- for (InProgressPool ppool : inProgressPoolList) {
- if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
- ppool.handleContainerReport(datanodeDetails, containerReport);
- return;
- }
- }
- // TODO: Decide if we can do anything else with this report.
- LOG.debug("Discarding the container report for pool {}. " +
- "That pool is not currently in the pool reconciliation process." +
- " Container Name: {}", poolName, datanodeDetails);
- } catch (SCMException e) {
- LOG.warn("Skipping processing container report from datanode {}, "
- + "cause: failed to get the corresponding node pool",
- datanodeDetails.toString(), e);
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- }
-
- /**
- * Get in process pool list, used for testing.
- * @return List of InProgressPool
- */
- @VisibleForTesting
- public List<InProgressPool> getInProcessPoolList() {
- return inProgressPoolList;
- }
-
- /**
- * Shutdown the Container Replication Manager.
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- setExit();
- HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 4b54731..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
- public static final Logger LOG =
- LoggerFactory.getLogger(InProgressPool.class);
-
- private final PeriodicPool pool;
- private final NodeManager nodeManager;
- private final NodePoolManager poolManager;
- private final ExecutorService executorService;
- private final Map<Long, Integer> containerCountMap;
- private final Map<UUID, Boolean> processedNodeSet;
- private final long startTime;
- private ProgressStatus status;
- private AtomicInteger nodeCount;
- private AtomicInteger nodeProcessed;
- private AtomicInteger containerProcessedCount;
- private long maxWaitTime;
- /**
- * Constructs an pool that is being processed.
- * @param maxWaitTime - Maximum wait time in milliseconds.
- * @param pool - Pool that we are working against
- * @param nodeManager - Nodemanager
- * @param poolManager - pool manager
- * @param executorService - Shared Executor service.
- */
- InProgressPool(long maxWaitTime, PeriodicPool pool,
- NodeManager nodeManager, NodePoolManager poolManager,
- ExecutorService executorService) {
- Preconditions.checkNotNull(pool);
- Preconditions.checkNotNull(nodeManager);
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(executorService);
- Preconditions.checkArgument(maxWaitTime > 0);
- this.pool = pool;
- this.nodeManager = nodeManager;
- this.poolManager = poolManager;
- this.executorService = executorService;
- this.containerCountMap = new ConcurrentHashMap<>();
- this.processedNodeSet = new ConcurrentHashMap<>();
- this.maxWaitTime = maxWaitTime;
- startTime = Time.monotonicNow();
- }
-
- /**
- * Returns periodic pool.
- *
- * @return PeriodicPool
- */
- public PeriodicPool getPool() {
- return pool;
- }
-
- /**
- * We are done if we have got reports from all nodes or we have
- * done waiting for the specified time.
- *
- * @return true if we are done, false otherwise.
- */
- public boolean isDone() {
- return (nodeCount.get() == nodeProcessed.get()) ||
- (this.startTime + this.maxWaitTime) > Time.monotonicNow();
- }
-
- /**
- * Gets the number of containers processed.
- *
- * @return int
- */
- public int getContainerProcessedCount() {
- return containerProcessedCount.get();
- }
-
- /**
- * Returns the start time in milliseconds.
- *
- * @return - Start Time.
- */
- public long getStartTime() {
- return startTime;
- }
-
- /**
- * Get the number of nodes in this pool.
- *
- * @return - node count
- */
- public int getNodeCount() {
- return nodeCount.get();
- }
-
- /**
- * Get the number of nodes that we have already processed container reports
- * from.
- *
- * @return - Processed count.
- */
- public int getNodeProcessed() {
- return nodeProcessed.get();
- }
-
- /**
- * Returns the current status.
- *
- * @return Status
- */
- public ProgressStatus getStatus() {
- return status;
- }
-
- /**
- * Starts the reconciliation process for all the nodes in the pool.
- */
- public void startReconciliation() {
- List<DatanodeDetails> datanodeDetailsList =
- this.poolManager.getNodes(pool.getPoolName());
- if (datanodeDetailsList.size() == 0) {
- LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
- pool.getPoolName());
- this.status = ProgressStatus.Error;
- return;
- }
-
- nodeProcessed = new AtomicInteger(0);
- containerProcessedCount = new AtomicInteger(0);
- nodeCount = new AtomicInteger(0);
- this.status = ProgressStatus.InProgress;
- this.getPool().setLastProcessedTime(Time.monotonicNow());
- }
-
- /**
- * Queues a container Report for handling. This is done in a worker thread
- * since decoding a container report might be compute intensive . We don't
- * want to block since we have asked for bunch of container reports
- * from a set of datanodes.
- *
- * @param containerReport - ContainerReport
- */
- public void handleContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto containerReport) {
- if (status == ProgressStatus.InProgress) {
- executorService.submit(processContainerReport(datanodeDetails,
- containerReport));
- } else {
- LOG.debug("Cannot handle container report when the pool is in {} status.",
- status);
- }
- }
-
- private Runnable processContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports) {
- return () -> {
- if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
- (k) -> true)) {
- nodeProcessed.incrementAndGet();
- LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
- datanodeDetails.getUuid());
- for (ContainerInfo info : reports.getReportsList()) {
- containerProcessedCount.incrementAndGet();
- LOG.debug("Total Containers processed: {} Container Name: {}",
- containerProcessedCount.get(), info.getContainerID());
-
- // Update the container map with count + 1 if the key exists or
- // update the map with 1. Since this is a concurrentMap the
- // computation and update is atomic.
- containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
- }
- }
- };
- }
-
- /**
- * Filter the containers based on specific rules.
- *
- * @param predicate -- Predicate to filter by
- * @return A list of map entries.
- */
- public List<Map.Entry<Long, Integer>> filterContainer(
- Predicate<Map.Entry<Long, Integer>> predicate) {
- return containerCountMap.entrySet().stream()
- .filter(predicate).collect(Collectors.toList());
- }
-
- /**
- * Used only for testing, calling this will abort container report
- * processing. This is very dangerous call and should not be made by any users
- */
- @VisibleForTesting
- public void setDoneProcessing() {
- nodeProcessed.set(nodeCount.get());
- }
-
- /**
- * Returns the pool name.
- *
- * @return Name of the pool.
- */
- String getPoolName() {
- return pool.getPoolName();
- }
-
- public void finalizeReconciliation() {
- status = ProgressStatus.Done;
- //TODO: Add finalizing logic. This is where actual reconciliation happens.
- }
-
- /**
- * Current status of the computing replication status.
- */
- public enum ProgressStatus {
- InProgress, Done, Error
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index ef28aa7..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
- private final String poolName;
- private long lastProcessedTime;
- private AtomicLong totalProcessedCount;
-
- /**
- * Constructs a periodic pool.
- *
- * @param poolName - Name of the pool
- */
- public PeriodicPool(String poolName) {
- this.poolName = poolName;
- lastProcessedTime = 0;
- totalProcessedCount = new AtomicLong(0);
- }
-
- /**
- * Get pool Name.
- * @return PoolName
- */
- public String getPoolName() {
- return poolName;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(PeriodicPool o) {
- return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- PeriodicPool that = (PeriodicPool) o;
-
- return poolName.equals(that.poolName);
- }
-
- @Override
- public int hashCode() {
- return poolName.hashCode();
- }
-
- /**
- * Returns the Total Times we have processed this pool.
- *
- * @return processed count.
- */
- public long getTotalProcessedCount() {
- return totalProcessedCount.get();
- }
-
- /**
- * Gets the last time we processed this pool.
- * @return time in milliseconds
- */
- public long getLastProcessedTime() {
- return this.lastProcessedTime;
- }
-
-
- /**
- * Sets the last processed time.
- *
- * @param lastProcessedTime - Long in milliseconds.
- */
-
- public void setLastProcessedTime(long lastProcessedTime) {
- this.lastProcessedTime = lastProcessedTime;
- }
-
- /*
- * Increments the total processed count.
- */
- public void incTotalProcessedCount() {
- this.totalProcessedCount.incrementAndGet();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
deleted file mode 100644
index 7bbe2ef..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 4392633..72d7e94 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,12 +124,6 @@ public interface NodeManager extends StorageContainerNodeProtocol,
SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
/**
- * Returns the NodePoolManager associated with the NodeManager.
- * @return NodePoolManager
- */
- NodePoolManager getNodePoolManager();
-
- /**
* Wait for the heartbeat is processed by NodeManager.
* @return true if heartbeat has been processed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
deleted file mode 100644
index 46faf9ca..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
- /**
- * Add a node to a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- */
- void addNode(String pool, DatanodeDetails node) throws IOException;
-
- /**
- * Remove a node from a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- * @throws SCMException
- */
- void removeNode(String pool, DatanodeDetails node)
- throws SCMException;
-
- /**
- * Get a list of known node pools.
- * @return a list of known node pool names or an empty list if not node pool
- * is defined.
- */
- List<String> getNodePools();
-
- /**
- * Get all nodes of a node pool given the name of the node pool.
- * @param pool - name of the node pool.
- * @return a list of datanode ids or an empty list if the node pool was not
- * found.
- */
- List<DatanodeDetails> getNodes(String pool);
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeDetails - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- */
- String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fc8b013..adca8ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import com.google.protobuf.GeneratedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -159,7 +158,6 @@ public class SCMNodeManager
private ObjectName nmInfoBean;
// Node pool manager.
- private final SCMNodePoolManager nodePoolManager;
private final StorageContainerManager scmManager;
public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -210,7 +208,6 @@ public class SCMNodeManager
registerMXBean();
- this.nodePoolManager = new SCMNodePoolManager(conf);
this.scmManager = scmManager;
}
@@ -682,7 +679,6 @@ public class SCMNodeManager
@Override
public void close() throws IOException {
unregisterMXBean();
- nodePoolManager.close();
executorService.shutdown();
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -760,20 +756,6 @@ public class SCMNodeManager
LOG.info("Leaving startup chill mode.");
}
- // TODO: define node pool policy for non-default node pool.
- // For now, all nodes are added to the "DefaultNodePool" upon registration
- // if it has not been added to any node pool yet.
- try {
- if (nodePoolManager.getNodePool(datanodeDetails) == null) {
- nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
- datanodeDetails);
- }
- } catch (IOException e) {
- // TODO: make sure registration failure is handled correctly.
- return RegisteredCommand.newBuilder()
- .setErrorCode(ErrorCode.errorNodeNotPermitted)
- .build();
- }
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
LOG.info("Data node with ID: {} Registered.",
@@ -860,11 +842,6 @@ public class SCMNodeManager
}
@Override
- public NodePoolManager getNodePoolManager() {
- return nodePoolManager;
- }
-
- @Override
public Map<String, Integer> getNodeCount() {
Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
for(NodeState state : NodeState.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index faf330e..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMNodePoolManager.class);
- private static final List<DatanodeDetails> EMPTY_NODE_LIST =
- new ArrayList<>();
- private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
- public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
- // DB that saves the node to node pool mapping.
- private MetadataStore nodePoolStore;
-
- // In-memory node pool to nodes mapping
- private HashMap<String, Set<DatanodeDetails>> nodePools;
-
- // Read-write lock for nodepool operations
- private ReadWriteLock lock;
-
- /**
- * Construct SCMNodePoolManager class that manages node to node pool mapping.
- * @param conf - configuration.
- * @throws IOException
- */
- public SCMNodePoolManager(final OzoneConfiguration conf)
- throws IOException {
- final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- File metaDir = getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
- File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
- nodePoolStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(nodePoolDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
- nodePools = new HashMap<>();
- lock = new ReentrantReadWriteLock();
- init();
- }
-
- /**
- * Initialize the in-memory store based on persist store from level db.
- * No lock is needed as init() is only invoked by constructor.
- * @throws SCMException
- */
- private void init() throws SCMException {
- try {
- nodePoolStore.iterate(null, (key, value) -> {
- try {
- DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
- HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
- String poolName = DFSUtil.bytes2String(value);
-
- Set<DatanodeDetails> nodePool = null;
- if (nodePools.containsKey(poolName)) {
- nodePool = nodePools.get(poolName);
- } else {
- nodePool = new HashSet<>();
- nodePools.put(poolName, nodePool);
- }
- nodePool.add(nodeId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding node: {} to node pool: {}",
- nodeId, poolName);
- }
- } catch (IOException e) {
- LOG.warn("Can't add a datanode to node pool, continue next...");
- }
- return true;
- });
- } catch (IOException e) {
- LOG.error("Loading node pool error " + e);
- throw new SCMException("Failed to load node pool",
- FAILED_TO_LOAD_NODEPOOL);
- }
- }
-
- /**
- * Add a datanode to a node pool.
- * @param pool - name of the node pool.
- * @param node - name of the datanode.
- */
- @Override
- public void addNode(final String pool, final DatanodeDetails node)
- throws IOException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // add to the persistent store
- nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
- DFSUtil.string2Bytes(pool));
-
- // add to the in-memory store
- Set<DatanodeDetails> nodePool = null;
- if (nodePools.containsKey(pool)) {
- nodePool = nodePools.get(pool);
- } else {
- nodePool = new HashSet<DatanodeDetails>();
- nodePools.put(pool, nodePool);
- }
- nodePool.add(node);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Remove a datanode from a node pool.
- * @param pool - name of the node pool.
- * @param node - datanode id.
- * @throws SCMException
- */
- @Override
- public void removeNode(final String pool, final DatanodeDetails node)
- throws SCMException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // Remove from the persistent store
- byte[] kName = node.getProtoBufMessage().toByteArray();
- byte[] kData = nodePoolStore.get(kName);
- if (kData == null) {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL);
- }
- nodePoolStore.delete(kName);
-
- // Remove from the in-memory store
- if (nodePools.containsKey(pool)) {
- Set<DatanodeDetails> nodePool = nodePools.get(pool);
- nodePool.remove(node);
- } else {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL);
- }
- } catch (IOException e) {
- throw new SCMException("Failed to remove node " + node.toString()
- + " from node pool " + pool, e,
- SCMException.ResultCodes.IO_EXCEPTION);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Get all the node pools.
- * @return all the node pools.
- */
- @Override
- public List<String> getNodePools() {
- lock.readLock().lock();
- try {
- if (!nodePools.isEmpty()) {
- return nodePools.keySet().stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODEPOOL_LIST;
- }
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Get all datanodes of a specific node pool.
- * @param pool - name of the node pool.
- * @return all datanodes of the specified node pool.
- */
- @Override
- public List<DatanodeDetails> getNodes(final String pool) {
- Preconditions.checkNotNull(pool, "pool name is null");
- if (nodePools.containsKey(pool)) {
- return nodePools.get(pool).stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODE_LIST;
- }
- }
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeDetails - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- * TODO: Put this in a in-memory map if performance is an issue.
- */
- @Override
- public String getNodePool(final DatanodeDetails datanodeDetails)
- throws SCMException {
- Preconditions.checkNotNull(datanodeDetails, "node is null");
- try {
- byte[] result = nodePoolStore.get(
- datanodeDetails.getProtoBufMessage().toByteArray());
- return result == null ? null : DFSUtil.bytes2String(result);
- } catch (IOException e) {
- throw new SCMException("Failed to get node pool for node "
- + datanodeDetails.toString(), e,
- SCMException.ResultCodes.IO_EXCEPTION);
- }
- }
-
- /**
- * Close node pool level db store.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- nodePoolStore.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 8c59462..80b5d6e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -273,11 +272,6 @@ public class MockNodeManager implements NodeManager {
return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
}
- @Override
- public NodePoolManager getNodePoolManager() {
- return Mockito.mock(NodePoolManager.class);
- }
-
/**
* Used for testing.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
deleted file mode 100644
index 8f412de..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for SCM node pool manager.
- */
-public class TestSCMNodePoolManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSCMNodePoolManager.class);
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- private final File testDir = PathUtils.getTestDir(
- TestSCMNodePoolManager.class);
-
- SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
- throws IOException {
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
- testDir.getAbsolutePath());
- conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
- return new SCMNodePoolManager(conf);
- }
-
- /**
- * Test default node pool.
- *
- * @throws IOException
- */
- @Test
- public void testDefaultNodePool() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- try {
- final String defaultPool = "DefaultPool";
- NodePoolManager npMgr = createNodePoolManager(conf);
-
- final int nodeCount = 4;
- final List<DatanodeDetails> nodes = TestUtils
- .getListOfDatanodeDetails(nodeCount);
- assertEquals(0, npMgr.getNodePools().size());
- for (DatanodeDetails node: nodes) {
- npMgr.addNode(defaultPool, node);
- }
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-
- DatanodeDetails nodeRemoved = nodes.remove(2);
- npMgr.removeNode(defaultPool, nodeRemoved);
- List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
- assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
-
- List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
- assertEquals(0, nonExistSet.size());
- } finally {
- FileUtil.fullyDelete(testDir);
- }
- }
-
-
- /**
- * Test default node pool reload.
- *
- * @throws IOException
- */
- @Test
- public void testDefaultNodePoolReload() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- final String defaultPool = "DefaultPool";
- final int nodeCount = 4;
- final List<DatanodeDetails> nodes = TestUtils
- .getListOfDatanodeDetails(nodeCount);
-
- try {
- try {
- SCMNodePoolManager npMgr = createNodePoolManager(conf);
- assertEquals(0, npMgr.getNodePools().size());
- for (DatanodeDetails node : nodes) {
- npMgr.addNode(defaultPool, node);
- }
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
- npMgr.close();
- } finally {
- LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
- " and close.");
- }
-
- // try reload with a new NodePoolManager instance
- try {
- SCMNodePoolManager npMgr = createNodePoolManager(conf);
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
- } finally {
- LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
- }
- } finally {
- FileUtil.fullyDelete(testDir);
- }
- }
-
- /**
- * Compare and verify that two datanode lists are equal.
- * @param list1 - datanode list 1.
- * @param list2 - datanode list 2.
- */
- private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
- List<DatanodeDetails> list2) {
- assertEquals(list1.size(), list2.size());
- Collections.sort(list1);
- Collections.sort(list2);
- assertTrue(ListUtils.isEqualList(list1, list2));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 072d821..1a4dcd7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -201,10 +200,6 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
- @Override
- public NodePoolManager getNodePoolManager() {
- return Mockito.mock(NodePoolManager.class);
- }
/**
* Wait for the heartbeat is processed by NodeManager.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
deleted file mode 100644
index ffcd752..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.testutils;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Pool Manager replication mock.
- */
-public class ReplicationNodePoolManagerMock implements NodePoolManager {
-
- private final Map<DatanodeDetails, String> nodeMemberShip;
-
- /**
- * A node pool manager for testing.
- */
- public ReplicationNodePoolManagerMock() {
- nodeMemberShip = new HashMap<>();
- }
-
- /**
- * Add a node to a node pool.
- *
- * @param pool - name of the node pool.
- * @param node - data node.
- */
- @Override
- public void addNode(String pool, DatanodeDetails node) {
- nodeMemberShip.put(node, pool);
- }
-
- /**
- * Remove a node from a node pool.
- *
- * @param pool - name of the node pool.
- * @param node - data node.
- * @throws SCMException
- */
- @Override
- public void removeNode(String pool, DatanodeDetails node)
- throws SCMException {
- nodeMemberShip.remove(node);
-
- }
-
- /**
- * Get a list of known node pools.
- *
- * @return a list of known node pool names or an empty list if not node pool
- * is defined.
- */
- @Override
- public List<String> getNodePools() {
- Set<String> poolSet = new HashSet<>();
- for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
- poolSet.add(entry.getValue());
- }
- return new ArrayList<>(poolSet);
-
- }
-
- /**
- * Get all nodes of a node pool given the name of the node pool.
- *
- * @param pool - name of the node pool.
- * @return a list of datanode ids or an empty list if the node pool was not
- * found.
- */
- @Override
- public List<DatanodeDetails> getNodes(String pool) {
- Set<DatanodeDetails> datanodeSet = new HashSet<>();
- for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
- if (entry.getValue().equals(pool)) {
- datanodeSet.add(entry.getKey());
- }
- }
- return new ArrayList<>(datanodeSet);
- }
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- *
- * @param datanodeDetails DatanodeDetails.
- * @return node pool name if it has been assigned. null if the node has not
- * been assigned to any node pool yet.
- */
- @Override
- public String getNodePool(DatanodeDetails datanodeDetails) {
- return nodeMemberShip.get(datanodeDetails);
- }
-
- /**
- * Closes this stream and releases any system resources associated
- * with it. If the stream is already closed then invoking this
- * method has no effect.
- * <p>
- * <p> As noted in {@link AutoCloseable#close()}, cases where the
- * close may fail require careful attention. It is strongly advised
- * to relinquish the underlying resources and to internally
- * <em>mark</em> the {@code Closeable} as closed, prior to throwing
- * the {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 4d70af8..b4ed2b1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,12 +51,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* This class tests the CLI that transforms container into SQLite DB files.
@@ -177,34 +174,6 @@ public class TestContainerSQLCli {
}
@Test
- public void testConvertNodepoolDB() throws Exception {
- String dbOutPath = GenericTestUtils.getTempPath(
- UUID.randomUUID() + "/out_sql.db");
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
- String dbPath = dbRootPath + "/" + NODEPOOL_DB;
- String[] args = {"-p", dbPath, "-o", dbOutPath};
-
- cli.run(args);
-
- // verify the sqlite db
- HashMap<String, String> expectedPool = new HashMap<>();
- for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
- expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
- }
- Connection conn = connectDB(dbOutPath);
- String sql = "SELECT * FROM nodePool";
- ResultSet rs = executeQuery(conn, sql);
- while(rs.next()) {
- String datanodeUUID = rs.getString("datanodeUUID");
- String poolName = rs.getString("poolName");
- assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
- }
- assertEquals(0, expectedPool.size());
-
- Files.delete(Paths.get(dbOutPath));
- }
-
- @Test
public void testConvertContainerDB() throws Exception {
String dbOutPath = GenericTestUtils.getTempPath(
UUID.randomUUID() + "/out_sql.db");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8184. Too many metrics if
containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator.
Contributed by Yufei Gu
Posted by gi...@apache.org.
YARN-8184. Too many metrics if containerLocalizer/ResourceLocalizationService uses ReadWriteDiskValidator. Contributed by Yufei Gu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cdce86d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cdce86d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cdce86d
Branch: refs/heads/HADOOP-15461
Commit: 1cdce86d33d4b73ba6dd4136c966eb7e822b6f36
Parents: ae05562
Author: Yufei Gu <yu...@apple.com>
Authored: Fri Jun 22 14:02:32 2018 -0700
Committer: Yufei Gu <yu...@apple.com>
Committed: Fri Jun 22 14:03:55 2018 -0700
----------------------------------------------------------------------
.../containermanager/localizer/ContainerLocalizer.java | 5 +----
.../containermanager/localizer/ResourceLocalizationService.java | 5 +----
2 files changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 6a384ae..c034369 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -132,10 +132,7 @@ public class ContainerLocalizer {
this.recordFactory = recordFactory;
this.conf = new YarnConfiguration();
this.diskValidator = DiskValidatorFactory.getInstance(
- conf.get(YarnConfiguration.DISK_VALIDATOR,
- YarnConfiguration.DEFAULT_DISK_VALIDATOR));
- LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
- " is loaded.");
+ YarnConfiguration.DEFAULT_DISK_VALIDATOR);
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cdce86d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index ddae2ae..3f0a6fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -264,10 +264,7 @@ public class ResourceLocalizationService extends CompositeService
}
diskValidator = DiskValidatorFactory.getInstance(
- conf.get(YarnConfiguration.DISK_VALIDATOR,
- YarnConfiguration.DEFAULT_DISK_VALIDATOR));
- LOG.info("Disk Validator: " + YarnConfiguration.DISK_VALIDATOR +
- " is loaded.");
+ YarnConfiguration.DEFAULT_DISK_VALIDATOR);
cacheTargetSize =
conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_TARGET_SIZE_MB, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB) << 20;
cacheCleanupPeriod =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-15550. Avoid static
initialization of ObjectMappers
Posted by gi...@apache.org.
HADOOP-15550. Avoid static initialization of ObjectMappers
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a3c6e9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a3c6e9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a3c6e9c
Branch: refs/heads/HADOOP-15461
Commit: 7a3c6e9c3cd9ffdc71946fd12f5c3d59718c4939
Parents: c687a66
Author: Todd Lipcon <to...@apache.org>
Authored: Mon Jun 25 15:36:45 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Mon Jun 25 15:36:45 2018 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 7 ++----
.../web/DelegationTokenAuthenticator.java | 8 ++-----
.../apache/hadoop/util/HttpExceptionUtils.java | 12 ++--------
.../apache/hadoop/util/JsonSerialization.java | 24 ++++++++++++++++++++
.../crypto/key/kms/server/KMSJSONWriter.java | 6 ++---
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 7 ++----
...onfRefreshTokenBasedAccessTokenProvider.java | 8 +++----
.../CredentialBasedAccessTokenProvider.java | 8 +++----
.../apache/hadoop/mapreduce/JobSubmitter.java | 8 +++----
.../hadoop/fs/azure/security/JsonUtils.java | 4 ++--
10 files changed, 45 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index edbf897..7b46075 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
@@ -79,7 +80,6 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -132,9 +132,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private final ValueQueue<EncryptedKeyVersion> encKeyVersionQueue;
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
-
private final Text dtService;
// Allow fallback to default kms server port 9600 for certain tests that do
@@ -237,7 +234,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
private static void writeJson(Object obj, OutputStream os)
throws IOException {
Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
- WRITER.writeValue(writer, obj);
+ JsonSerialization.writer().writeValue(writer, obj);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 617773b..0ae2af3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.security.token.delegation.web;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.net.NetUtils;
@@ -31,6 +29,7 @@ import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.HttpExceptionUtils;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,9 +55,6 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
private static final String CONTENT_TYPE = "Content-Type";
private static final String APPLICATION_JSON_MIME = "application/json";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
-
private static final String HTTP_GET = "GET";
private static final String HTTP_PUT = "PUT";
@@ -328,7 +324,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
if (contentType != null &&
contentType.contains(APPLICATION_JSON_MIME)) {
try {
- ret = READER.readValue(conn.getInputStream());
+ ret = JsonSerialization.mapReader().readValue(conn.getInputStream());
} catch (Exception ex) {
throw new AuthenticationException(String.format(
"'%s' did not handle the '%s' delegation token operation: %s",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
index cdb8112..50be1c3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HttpExceptionUtils.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.util;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -56,11 +53,6 @@ public class HttpExceptionUtils {
private static final String ENTER = System.getProperty("line.separator");
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
-
/**
* Creates a HTTP servlet response serializing the exception in it as JSON.
*
@@ -82,7 +74,7 @@ public class HttpExceptionUtils {
Map<String, Object> jsonResponse = new LinkedHashMap<String, Object>();
jsonResponse.put(ERROR_JSON, json);
Writer writer = response.getWriter();
- WRITER.writeValue(writer, jsonResponse);
+ JsonSerialization.writer().writeValue(writer, jsonResponse);
writer.flush();
}
@@ -150,7 +142,7 @@ public class HttpExceptionUtils {
InputStream es = null;
try {
es = conn.getErrorStream();
- Map json = READER.readValue(es);
+ Map json = JsonSerialization.mapReader().readValue(es);
json = (Map) json.get(ERROR_JSON);
String exClass = (String) json.get(ERROR_CLASSNAME_JSON);
String exMsg = (String) json.get(ERROR_MESSAGE_JSON);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
index 86c4df6..cbc8560 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -25,14 +25,18 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Map;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.base.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +69,26 @@ public class JsonSerialization<T> {
private final Class<T> classType;
private final ObjectMapper mapper;
+ private static final ObjectWriter WRITER =
+ new ObjectMapper().writerWithDefaultPrettyPrinter();
+
+ private static final ObjectReader MAP_READER =
+ new ObjectMapper().readerFor(Map.class);
+
+ /**
+ * @return an ObjectWriter which pretty-prints its output
+ */
+ public static ObjectWriter writer() {
+ return WRITER;
+ }
+
+ /**
+ * @return an ObjectReader which returns simple Maps.
+ */
+ public static ObjectReader mapReader() {
+ return MAP_READER;
+ }
+
/**
* Create an instance bound to a specific type.
* @param classType class to marshall
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
index f826572..b9b8d9c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONWriter.java
@@ -17,10 +17,9 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.JsonSerialization;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
@@ -67,8 +66,7 @@ public class KMSJSONWriter implements MessageBodyWriter<Object> {
OutputStream outputStream) throws IOException, WebApplicationException {
Writer writer = new OutputStreamWriter(outputStream, Charset
.forName("UTF-8"));
- ObjectMapper jsonMapper = new ObjectMapper();
- jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
+ JsonSerialization.writer().writeValue(writer, obj);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 673acd6..ec60a18 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -56,8 +56,6 @@ import java.util.concurrent.TimeUnit;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.hadoop.conf.Configuration;
@@ -121,6 +119,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenSelector;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.StringUtils;
@@ -172,8 +171,6 @@ public class WebHdfsFileSystem extends FileSystem
private boolean disallowFallbackToInsecureCluster;
private String restCsrfCustomHeader;
private Set<String> restCsrfMethodsToIgnore;
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
private DFSOpsCountStatistics storageStatistics;
@@ -476,7 +473,7 @@ public class WebHdfsFileSystem extends FileSystem
+ "\" (parsed=\"" + parsed + "\")");
}
}
- return READER.readValue(in);
+ return JsonSerialization.mapReader().readValue(in);
} finally {
in.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
index c6ebdd6..3e3fbfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpStatus;
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
@InterfaceStability.Evolving
public class ConfRefreshTokenBasedAccessTokenProvider
extends AccessTokenProvider {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
public static final String OAUTH_REFRESH_TOKEN_KEY
= "dfs.webhdfs.oauth2.refresh.token";
@@ -129,7 +126,8 @@ public class ConfRefreshTokenBasedAccessTokenProvider
+ responseBody.code() + ", text = " + responseBody.toString());
}
- Map<?, ?> response = READER.readValue(responseBody.body().string());
+ Map<?, ?> response = JsonSerialization.mapReader().readValue(
+ responseBody.body().string());
String newExpiresIn = response.get(EXPIRES_IN).toString();
accessTokenTimer.setExpiresIn(newExpiresIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
index 5c629e0..bfd7055 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hdfs.web.oauth2;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.RequestBody;
@@ -28,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.Timer;
import org.apache.http.HttpStatus;
@@ -55,8 +54,6 @@ import static org.apache.hadoop.hdfs.web.oauth2.Utils.notNull;
@InterfaceStability.Evolving
public abstract class CredentialBasedAccessTokenProvider
extends AccessTokenProvider {
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
public static final String OAUTH_CREDENTIAL_KEY
= "dfs.webhdfs.oauth2.credential";
@@ -123,7 +120,8 @@ public abstract class CredentialBasedAccessTokenProvider
+ responseBody.code() + ", text = " + responseBody.toString());
}
- Map<?, ?> response = READER.readValue(responseBody.body().string());
+ Map<?, ?> response = JsonSerialization.mapReader().readValue(
+ responseBody.body().string());
String newExpiresIn = response.get(EXPIRES_IN).toString();
timer.setExpiresIn(newExpiresIn);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 246986f..ebf9341 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -34,8 +34,6 @@ import javax.crypto.SecretKey;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +59,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.JsonSerialization;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -71,8 +70,6 @@ import com.google.common.base.Charsets;
class JobSubmitter {
protected static final Logger LOG =
LoggerFactory.getLogger(JobSubmitter.class);
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(Map.class);
private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
private static final int SHUFFLE_KEY_LENGTH = 64;
private FileSystem jtFs;
@@ -406,7 +403,8 @@ class JobSubmitter {
try {
// read JSON
- Map<String, String> nm = READER.readValue(new File(localFileName));
+ Map<String, String> nm = JsonSerialization.mapReader().readValue(
+ new File(localFileName));
for(Map.Entry<String, String> ent: nm.entrySet()) {
credentials.addSecretKey(new Text(ent.getKey()), ent.getValue()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a3c6e9c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
index 20dd470..9c40325 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/JsonUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azure.security;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.util.JsonSerialization;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,8 +38,7 @@ public final class JsonUtils {
public static Map<?, ?> parse(final String jsonString) throws IOException {
try {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readerFor(Map.class).readValue(jsonString);
+ return JsonSerialization.mapReader().readValue(jsonString);
} catch (Exception e) {
LOG.debug("JSON Parsing exception: {} while parsing {}", e.getMessage(),
jsonString);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-14313. Replace/improve
Hadoop's byte[] comparator. Contributed by Vikas Vishwakarma.
Posted by gi...@apache.org.
HADOOP-14313. Replace/improve Hadoop's byte[] comparator. Contributed by Vikas Vishwakarma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddbff7c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddbff7c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddbff7c8
Branch: refs/heads/HADOOP-15461
Commit: ddbff7c8d3f1851e5c5fa9bc33637e859d7d8ccf
Parents: 2b2399d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jun 28 14:58:40 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Jun 28 14:58:40 2018 +0900
----------------------------------------------------------------------
NOTICE.txt | 8 ++++
.../apache/hadoop/io/FastByteComparisons.java | 44 ++++++++------------
2 files changed, 25 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddbff7c8/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
index 95a670d..a53f13c 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -196,6 +196,14 @@ by Google Inc, which can be obtained at:
* HOMEPAGE:
* http://code.google.com/p/snappy/
+This product contains a modified portion of UnsignedBytes LexicographicalComparator
+from Guava v21 project by Google Inc, which can be obtained at:
+
+ * LICENSE:
+ * license/COPYING (Apache License 2.0)
+ * HOMEPAGE:
+ * https://github.com/google/guava
+
This product optionally depends on 'JBoss Marshalling', an alternative Java
serialization API, which can be obtained at:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddbff7c8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
index a2903f8..5af6602 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/FastByteComparisons.java
@@ -26,7 +26,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sun.misc.Unsafe;
-import com.google.common.primitives.Longs;
import com.google.common.primitives.UnsignedBytes;
/**
@@ -195,52 +194,43 @@ abstract class FastByteComparisons {
length1 == length2) {
return 0;
}
+ final int stride = 8;
int minLength = Math.min(length1, length2);
- int minWords = minLength / Longs.BYTES;
+ int strideLimit = minLength & ~(stride - 1);
int offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET;
int offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET;
+ int i;
/*
* Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a
* time is no slower than comparing 4 bytes at a time even on 32-bit.
* On the other hand, it is substantially faster on 64-bit.
*/
- for (int i = 0; i < minWords * Longs.BYTES; i += Longs.BYTES) {
+ for (i = 0; i < strideLimit; i += stride) {
long lw = theUnsafe.getLong(buffer1, offset1Adj + (long) i);
long rw = theUnsafe.getLong(buffer2, offset2Adj + (long) i);
- long diff = lw ^ rw;
- if (diff != 0) {
+ if (lw != rw) {
if (!littleEndian) {
return lessThanUnsigned(lw, rw) ? -1 : 1;
}
- // Use binary search
- int n = 0;
- int y;
- int x = (int) diff;
- if (x == 0) {
- x = (int) (diff >>> 32);
- n = 32;
- }
-
- y = x << 16;
- if (y == 0) {
- n += 16;
- } else {
- x = y;
- }
-
- y = x << 8;
- if (y == 0) {
- n += 8;
- }
- return (int) (((lw >>> n) & 0xFFL) - ((rw >>> n) & 0xFFL));
+ /*
+ * We want to compare only the first index where left[index] !=
+ * right[index]. This corresponds to the least significant nonzero
+ * byte in lw ^ rw, since lw and rw are little-endian.
+ * Long.numberOfTrailingZeros(diff) tells us the least significant
+ * nonzero bit, and zeroing out the first three bits of L.nTZ gives
+ * us the shift to get that least significant nonzero byte. This
+ * comparison logic is based on UnsignedBytes from Guava v21
+ */
+ int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7;
+ return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF));
}
}
// The epilogue to cover the last (minLength % 8) elements.
- for (int i = minWords * Longs.BYTES; i < minLength; i++) {
+ for (; i < minLength; i++) {
int result = UnsignedBytes.compare(
buffer1[offset1 + i],
buffer2[offset2 + i]);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-8457. Compilation is broken
with -Pyarn-ui.
Posted by gi...@apache.org.
YARN-8457. Compilation is broken with -Pyarn-ui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ffe68a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ffe68a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ffe68a6
Branch: refs/heads/HADOOP-15461
Commit: 4ffe68a6f70ce01a5654da8991b4cdb35ae0bf1f
Parents: abc3e4b
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Jun 25 10:38:03 2018 -0700
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Jun 25 10:38:03 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ffe68a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
index 959e169..daf4462 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.bowerrc
@@ -1,4 +1,5 @@
{
"directory": "bower_components",
- "analytics": false
+ "analytics": false,
+ "registry": "https://registry.bower.io"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-15522. Deprecate
Shell#ReadLink by using native java code. Contributed by Giovanni Matteo
Fumarola.
Posted by gi...@apache.org.
HADOOP-15522. Deprecate Shell#ReadLink by using native java code. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75f23cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75f23cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75f23cbf
Branch: refs/heads/HADOOP-15461
Commit: 75f23cbf655c36d255d5961e03af9a8f86414404
Parents: 32372ea
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jun 11 13:14:34 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 13:08:57 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileUtil.java | 21 +++++++++++---------
.../main/java/org/apache/hadoop/util/Shell.java | 8 +++++++-
2 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75f23cbf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 61cb8d2..f3b5d58 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -196,22 +196,25 @@ public class FileUtil {
* a symlink.
*/
public static String readLink(File f) {
- /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
- * use getCanonicalPath in File to get the target of the symlink but that
- * does not indicate if the given path refers to a symlink.
- */
if (f == null) {
LOG.warn("Can not read a null symLink");
return "";
}
- try {
- return Shell.execCommand(
- Shell.getReadlinkCommand(f.toString())).trim();
- } catch (IOException x) {
- return "";
+ if (Files.isSymbolicLink(f.toPath())) {
+ java.nio.file.Path p = null;
+ try {
+ p = Files.readSymbolicLink(f.toPath());
+ } catch (Exception e) {
+ LOG.warn("Exception while reading the symbolic link "
+ + f.getAbsolutePath() + ". Exception= " + e.getMessage());
+ return "";
+ }
+ return p.toAbsolutePath().toString();
}
+ LOG.warn("The file " + f.getAbsolutePath() + " is not a symbolic link.");
+ return "";
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75f23cbf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index e902af0..691df63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -309,7 +309,13 @@ public abstract class Shell {
: new String[] { "ln", "-s", target, link };
}
- /** Return a command to read the target of the a symbolic link. */
+ /**
+ * Return a command to read the target of the a symbolic link.
+ *
+ * Deprecated and likely to be deleted in the near future. Please use
+ * FileUtil.symlink().
+ */
+ @Deprecated
public static String[] getReadlinkCommand(String link) {
return WINDOWS ?
new String[] { getWinUtilsPath(), "readlink", link }
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-8423. GPU does not get
released even though the application gets killed. (Sunil G via wangda)
Posted by gi...@apache.org.
YARN-8423. GPU does not get released even though the application gets killed. (Sunil G via wangda)
Change-Id: I570db7d60f8c6c21762dd618a9207d1107c486a0
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ada8f63d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ada8f63d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ada8f63d
Branch: refs/heads/HADOOP-15461
Commit: ada8f63d0b3739d245300461387b0516dc92ccf9
Parents: 62d83ca
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Jun 26 19:25:57 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Jun 26 19:25:57 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/Container.java | 6 ++
.../container/ContainerImpl.java | 11 ++++
.../resources/gpu/GpuResourceAllocator.java | 68 +++++++++++++++++++-
.../resources/gpu/GpuResourceHandlerImpl.java | 1 -
.../nodemanager/webapp/MockContainer.java | 3 +
5 files changed, 85 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 5d48d84..4912d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -113,4 +113,10 @@ public interface Container extends EventHandler<ContainerEvent> {
ResourceMappings getResourceMappings();
void sendPauseEvent(String description);
+
+ /**
+ * Verify container is in final states.
+ * @return true/false based on container's state
+ */
+ boolean isContainerInFinalStates();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 0541544..f76e682 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -2223,4 +2223,15 @@ public class ContainerImpl implements Container {
SlidingWindowRetryPolicy getRetryPolicy() {
return retryPolicy;
}
+
+ @Override
+ public boolean isContainerInFinalStates() {
+ ContainerState state = getContainerState();
+ return state == ContainerState.KILLING || state == ContainerState.DONE
+ || state == ContainerState.LOCALIZATION_FAILED
+ || state == ContainerState.CONTAINER_RESOURCES_CLEANINGUP
+ || state == ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL
+ || state == ContainerState.EXITED_WITH_FAILURE
+ || state == ContainerState.EXITED_WITH_SUCCESS;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
index 5bdffc3..81a9655 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -36,10 +37,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -54,6 +53,7 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.GPU_URI;
*/
public class GpuResourceAllocator {
final static Log LOG = LogFactory.getLog(GpuResourceAllocator.class);
+ private static final int WAIT_MS_PER_LOOP = 1000;
private Set<GpuDevice> allowedGpuDevices = new TreeSet<>();
private Map<GpuDevice, ContainerId> usedDevices = new TreeMap<>();
@@ -168,7 +168,44 @@ public class GpuResourceAllocator {
* @return allocation results.
* @throws ResourceHandlerException When failed to assign GPUs.
*/
- public synchronized GpuAllocation assignGpus(Container container)
+ public GpuAllocation assignGpus(Container container)
+ throws ResourceHandlerException {
+ GpuAllocation allocation = internalAssignGpus(container);
+
+ // Wait for a maximum of 120 seconds if no available GPU are there which
+ // are yet to be released.
+ final int timeoutMsecs = 120 * WAIT_MS_PER_LOOP;
+ int timeWaiting = 0;
+ while (allocation == null) {
+ if (timeWaiting >= timeoutMsecs) {
+ break;
+ }
+
+ // Sleep for 1 sec to ensure there are some free GPU devices which are
+ // getting released.
+ try {
+ LOG.info("Container : " + container.getContainerId()
+ + " is waiting for free GPU devices.");
+ Thread.sleep(WAIT_MS_PER_LOOP);
+ timeWaiting += WAIT_MS_PER_LOOP;
+ allocation = internalAssignGpus(container);
+ } catch (InterruptedException e) {
+ // On any interrupt, break the loop and continue execution.
+ break;
+ }
+ }
+
+ if(allocation == null) {
+ String message = "Could not get valid GPU device for container '" +
+ container.getContainerId()
+ + "' as some other containers might not releasing GPUs.";
+ LOG.warn(message);
+ throw new ResourceHandlerException(message);
+ }
+ return allocation;
+ }
+
+ private synchronized GpuAllocation internalAssignGpus(Container container)
throws ResourceHandlerException {
Resource requestedResource = container.getResource();
ContainerId containerId = container.getContainerId();
@@ -176,6 +213,14 @@ public class GpuResourceAllocator {
// Assign Gpus to container if requested some.
if (numRequestedGpuDevices > 0) {
if (numRequestedGpuDevices > getAvailableGpus()) {
+ // If there are some devices which are getting released, wait for few
+ // seconds to get it.
+ if (numRequestedGpuDevices <= getReleasingGpus() + getAvailableGpus()) {
+ return null;
+ }
+ }
+
+ if (numRequestedGpuDevices > getAvailableGpus()) {
throw new ResourceHandlerException(
getResourceHandlerExceptionMessage(numRequestedGpuDevices,
containerId));
@@ -211,6 +256,23 @@ public class GpuResourceAllocator {
return new GpuAllocation(null, allowedGpuDevices);
}
+ private synchronized long getReleasingGpus() {
+ long releasingGpus = 0;
+ Iterator<Map.Entry<GpuDevice, ContainerId>> iter = usedDevices.entrySet()
+ .iterator();
+ while (iter.hasNext()) {
+ ContainerId containerId = iter.next().getValue();
+ Container container;
+ if ((container = nmContext.getContainers().get(containerId)) != null) {
+ if (container.isContainerInFinalStates()) {
+ releasingGpus = releasingGpus + container.getResource()
+ .getResourceInformation(ResourceInformation.GPU_URI).getValue();
+ }
+ }
+ }
+ return releasingGpus;
+ }
+
/**
* Clean up all Gpus assigned to containerId
* @param containerId containerId
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 587fcb4..1184382 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada8f63d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 325709b..67dfef2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -255,4 +255,7 @@ public class MockContainer implements Container {
public void sendPauseEvent(String description) {
}
+ @Override public boolean isContainerInFinalStates() {
+ return false;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-8409. Fixed NPE in
ActiveStandbyElectorBasedElectorService. Contributed by Chandni Singh
Posted by gi...@apache.org.
YARN-8409. Fixed NPE in ActiveStandbyElectorBasedElectorService.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/384764cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/384764cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/384764cd
Branch: refs/heads/HADOOP-15461
Commit: 384764cdeac6490bc47fa0eb7b936baa4c0d3230
Parents: d3fa83a
Author: Eric Yang <ey...@apache.org>
Authored: Thu Jun 28 12:39:49 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Jun 28 12:39:49 2018 -0400
----------------------------------------------------------------------
.../apache/hadoop/ha/ActiveStandbyElector.java | 5 ++++-
.../apache/hadoop/ha/ZKFailoverController.java | 2 +-
.../resourcemanager/TestRMEmbeddedElector.java | 22 ++++++++++++++++++++
3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index a23fb71..d099ca7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -329,9 +329,12 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
* This recursively creates the znode as well as all of its parents.
*/
public synchronized void ensureParentZNode()
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, KeeperException {
Preconditions.checkState(!wantToBeInElection,
"ensureParentZNode() may not be called while in the election");
+ if (zkClient == null) {
+ createConnection();
+ }
String pathParts[] = znodeWorkingDir.split("/");
Preconditions.checkArgument(pathParts.length >= 1 &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index 9295288..f66e3c9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -269,7 +269,7 @@ public abstract class ZKFailoverController {
}
private int formatZK(boolean force, boolean interactive)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, KeeperException {
if (elector.parentZNodeExists()) {
if (!force && (!interactive || !confirmFormat())) {
return ERR_CODE_FORMAT_DENIED;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/384764cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
index 9d38149..8c03861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
@@ -22,8 +22,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.ServiceFailedException;
+import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -304,6 +306,26 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes {
verify(as, times(1)).transitionToStandby(any());
}
+ /**
+ * Test that active elector service triggers a fatal RM Event when connection
+ * to ZK fails. YARN-8409
+ */
+ @Test
+ public void testFailureToConnectToZookeeper() throws Exception {
+ stopServer();
+ Configuration myConf = new Configuration(conf);
+ ResourceManager rm = new MockRM(conf);
+
+ ActiveStandbyElectorBasedElectorService ees =
+ new ActiveStandbyElectorBasedElectorService(rm);
+ try {
+ ees.init(myConf);
+ Assert.fail("expect failure to connect to Zookeeper");
+ } catch (ServiceStateException sse) {
+ Assert.assertTrue(sse.getMessage().contains("ConnectionLoss"));
+ }
+ }
+
private class MockRMWithElector extends MockRM {
private long delayMs = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-15423. Merge fileCache and
dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor
Bota.
Posted by gi...@apache.org.
HADOOP-15423. Merge fileCache and dirCache into ine single cache in LocalMetadataStore. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c687a661
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c687a661
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c687a661
Branch: refs/heads/HADOOP-15461
Commit: c687a6617d73293019d8d91ac48bbfd2ccca3b40
Parents: a55d6bb
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Jun 25 11:04:34 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Mon Jun 25 14:59:41 2018 -0600
----------------------------------------------------------------------
.../fs/s3a/s3guard/LocalMetadataEntry.java | 81 ++++++
.../fs/s3a/s3guard/LocalMetadataStore.java | 247 +++++++++++--------
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 2 +-
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 33 ++-
4 files changed, 240 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
new file mode 100644
index 0000000..6040d67
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataEntry.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import javax.annotation.Nullable;
+
+/**
+ * LocalMetadataEntry is used to store entries in the cache of
+ * LocalMetadataStore. PathMetadata or dirListingMetadata can be null. The
+ * entry is not immutable.
+ */
+public final class LocalMetadataEntry {
+ @Nullable
+ private PathMetadata pathMetadata;
+ @Nullable
+ private DirListingMetadata dirListingMetadata;
+
+ LocalMetadataEntry(PathMetadata pmd){
+ pathMetadata = pmd;
+ dirListingMetadata = null;
+ }
+
+ LocalMetadataEntry(DirListingMetadata dlm){
+ pathMetadata = null;
+ dirListingMetadata = dlm;
+ }
+
+ public PathMetadata getFileMeta() {
+ return pathMetadata;
+ }
+
+ public DirListingMetadata getDirListingMeta() {
+ return dirListingMetadata;
+ }
+
+
+ public boolean hasPathMeta() {
+ return this.pathMetadata != null;
+ }
+
+ public boolean hasDirMeta() {
+ return this.dirListingMetadata != null;
+ }
+
+ public void setPathMetadata(PathMetadata pathMetadata) {
+ this.pathMetadata = pathMetadata;
+ }
+
+ public void setDirListingMetadata(DirListingMetadata dirListingMetadata) {
+ this.dirListingMetadata = dirListingMetadata;
+ }
+
+ @Override public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("LocalMetadataEntry{");
+ if(pathMetadata != null) {
+ sb.append("pathMetadata=" + pathMetadata.getFileStatus().getPath());
+ }
+ if(dirListingMetadata != null){
+ sb.append("; dirListingMetadata=" + dirListingMetadata.getPath());
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 95689e1..49981ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -37,13 +37,12 @@ import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
- * This is a local, in-memory, implementation of MetadataStore.
+ * This is a local, in-memory implementation of MetadataStore.
* This is <i>not</i> a coherent cache across processes. It is only
* locally-coherent.
*
@@ -56,12 +55,12 @@ import java.util.concurrent.TimeUnit;
* non-recursive removal of non-empty directories. It is assumed the caller
* already has to perform these sorts of checks.
*
- * Contains cache internally with time based eviction.
+ * Contains one cache internally with time based eviction.
*/
public class LocalMetadataStore implements MetadataStore {
public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
- public static final int DEFAULT_MAX_RECORDS = 128;
+ public static final int DEFAULT_MAX_RECORDS = 256;
public static final int DEFAULT_CACHE_ENTRY_TTL_MSEC = 10 * 1000;
/**
@@ -79,11 +78,8 @@ public class LocalMetadataStore implements MetadataStore {
public static final String CONF_CACHE_ENTRY_TTL =
"fs.metadatastore.local.ttl";
- /** Contains directories and files. */
- private Cache<Path, PathMetadata> fileCache;
-
- /** Contains directory listings. */
- private Cache<Path, DirListingMetadata> dirCache;
+ /** Contains directory and file listings. */
+ private Cache<Path, LocalMetadataEntry> localCache;
private FileSystem fs;
/* Null iff this FS does not have an associated URI host. */
@@ -116,8 +112,7 @@ public class LocalMetadataStore implements MetadataStore {
builder.expireAfterAccess(ttl, TimeUnit.MILLISECONDS);
}
- fileCache = builder.build();
- dirCache = builder.build();
+ localCache = builder.build();
}
@Override
@@ -155,8 +150,7 @@ public class LocalMetadataStore implements MetadataStore {
if (recursive) {
// Remove all entries that have this dir as path prefix.
- deleteEntryByAncestor(path, dirCache, tombstone);
- deleteEntryByAncestor(path, fileCache, tombstone);
+ deleteEntryByAncestor(path, localCache, tombstone);
}
}
@@ -170,7 +164,7 @@ public class LocalMetadataStore implements MetadataStore {
throws IOException {
Path path = standardize(p);
synchronized (this) {
- PathMetadata m = fileCache.getIfPresent(path);
+ PathMetadata m = getFileMeta(path);
if (wantEmptyDirectoryFlag && m != null &&
m.getFileStatus().isDirectory()) {
@@ -191,15 +185,15 @@ public class LocalMetadataStore implements MetadataStore {
* @return TRUE / FALSE if known empty / not-empty, UNKNOWN otherwise.
*/
private Tristate isEmptyDirectory(Path p) {
- DirListingMetadata dirMeta = dirCache.getIfPresent(p);
- return dirMeta.withoutTombstones().isEmpty();
+ DirListingMetadata dlm = getDirListingMeta(p);
+ return dlm.withoutTombstones().isEmpty();
}
@Override
public synchronized DirListingMetadata listChildren(Path p) throws
IOException {
Path path = standardize(p);
- DirListingMetadata listing = dirCache.getIfPresent(path);
+ DirListingMetadata listing = getDirListingMeta(path);
if (LOG.isDebugEnabled()) {
LOG.debug("listChildren({}) -> {}", path,
listing == null ? "null" : listing.prettyPrint());
@@ -211,6 +205,7 @@ public class LocalMetadataStore implements MetadataStore {
@Override
public void move(Collection<Path> pathsToDelete,
Collection<PathMetadata> pathsToCreate) throws IOException {
+ LOG.info("Move {} to {}", pathsToDelete, pathsToCreate);
Preconditions.checkNotNull(pathsToDelete, "pathsToDelete is null");
Preconditions.checkNotNull(pathsToCreate, "pathsToCreate is null");
@@ -258,7 +253,12 @@ public class LocalMetadataStore implements MetadataStore {
if (LOG.isDebugEnabled()) {
LOG.debug("put {} -> {}", path, meta.prettyPrint());
}
- fileCache.put(path, meta);
+ LocalMetadataEntry entry = localCache.getIfPresent(path);
+ if(entry == null){
+ entry = new LocalMetadataEntry(meta);
+ } else {
+ entry.setPathMetadata(meta);
+ }
/* Directory case:
* We also make sure we have an entry in the dirCache, so subsequent
@@ -271,27 +271,32 @@ public class LocalMetadataStore implements MetadataStore {
* saving round trips to underlying store for subsequent listStatus()
*/
- if (status.isDirectory()) {
- DirListingMetadata dir = dirCache.getIfPresent(path);
- if (dir == null) {
- dirCache.put(path, new DirListingMetadata(path, DirListingMetadata
- .EMPTY_DIR, false));
- }
+ // only create DirListingMetadata if the entry does not have one
+ if (status.isDirectory() && !entry.hasDirMeta()) {
+ DirListingMetadata dlm =
+ new DirListingMetadata(path, DirListingMetadata.EMPTY_DIR, false);
+ entry.setDirListingMetadata(dlm);
}
+ localCache.put(path, entry);
/* Update cached parent dir. */
Path parentPath = path.getParent();
if (parentPath != null) {
- DirListingMetadata parent = dirCache.getIfPresent(parentPath);
- if (parent == null) {
- /* Track this new file's listing in parent. Parent is not
- * authoritative, since there may be other items in it we don't know
- * about. */
- parent = new DirListingMetadata(parentPath,
- DirListingMetadata.EMPTY_DIR, false);
- dirCache.put(parentPath, parent);
+ LocalMetadataEntry parentMeta = localCache.getIfPresent(parentPath);
+ DirListingMetadata parentDirMeta =
+ new DirListingMetadata(parentPath, DirListingMetadata.EMPTY_DIR,
+ false);
+ parentDirMeta.put(status);
+
+ getDirListingMeta(parentPath);
+
+ if (parentMeta == null){
+ localCache.put(parentPath, new LocalMetadataEntry(parentDirMeta));
+ } else if (!parentMeta.hasDirMeta()) {
+ parentMeta.setDirListingMetadata(parentDirMeta);
+ } else {
+ parentMeta.getDirListingMeta().put(status);
}
- parent.put(status);
}
}
}
@@ -301,7 +306,13 @@ public class LocalMetadataStore implements MetadataStore {
if (LOG.isDebugEnabled()) {
LOG.debug("put dirMeta {}", meta.prettyPrint());
}
- dirCache.put(standardize(meta.getPath()), meta);
+ LocalMetadataEntry entry =
+ localCache.getIfPresent(standardize(meta.getPath()));
+ if(entry == null){
+ localCache.put(standardize(meta.getPath()), new LocalMetadataEntry(meta));
+ } else {
+ entry.setDirListingMetadata(meta);
+ }
put(meta.getListing());
}
@@ -319,8 +330,8 @@ public class LocalMetadataStore implements MetadataStore {
@Override
public void destroy() throws IOException {
- if (dirCache != null) {
- dirCache.invalidateAll();
+ if (localCache != null) {
+ localCache.invalidateAll();
}
}
@@ -330,42 +341,44 @@ public class LocalMetadataStore implements MetadataStore {
}
@Override
- public synchronized void prune(long modTime, String keyPrefix)
- throws IOException {
- Iterator<Map.Entry<Path, PathMetadata>> files =
- fileCache.asMap().entrySet().iterator();
- while (files.hasNext()) {
- Map.Entry<Path, PathMetadata> entry = files.next();
- if (expired(entry.getValue().getFileStatus(), modTime, keyPrefix)) {
- files.remove();
- }
- }
- Iterator<Map.Entry<Path, DirListingMetadata>> dirs =
- dirCache.asMap().entrySet().iterator();
- while (dirs.hasNext()) {
- Map.Entry<Path, DirListingMetadata> entry = dirs.next();
- Path path = entry.getKey();
- DirListingMetadata metadata = entry.getValue();
- Collection<PathMetadata> oldChildren = metadata.getListing();
- Collection<PathMetadata> newChildren = new LinkedList<>();
-
- for (PathMetadata child : oldChildren) {
- FileStatus status = child.getFileStatus();
- if (!expired(status, modTime, keyPrefix)) {
- newChildren.add(child);
- }
- }
- if (newChildren.size() != oldChildren.size()) {
- dirCache.put(path, new DirListingMetadata(path, newChildren, false));
- if (!path.isRoot()) {
- DirListingMetadata parent = null;
- parent = dirCache.getIfPresent(path.getParent());
- if (parent != null) {
- parent.setAuthoritative(false);
+ public synchronized void prune(long modTime, String keyPrefix) {
+ // prune files
+ // filter path_metadata (files), filter expired, remove expired
+ localCache.asMap().entrySet().stream()
+ .filter(entry -> entry.getValue().hasPathMeta())
+ .filter(entry -> expired(
+ entry.getValue().getFileMeta().getFileStatus(), modTime, keyPrefix))
+ .forEach(entry -> localCache.invalidate(entry.getKey()));
+
+
+ // prune dirs
+ // filter DIR_LISTING_METADATA, remove expired, remove authoritative bit
+ localCache.asMap().entrySet().stream()
+ .filter(entry -> entry.getValue().hasDirMeta())
+ .forEach(entry -> {
+ Path path = entry.getKey();
+ DirListingMetadata metadata = entry.getValue().getDirListingMeta();
+ Collection<PathMetadata> oldChildren = metadata.getListing();
+ Collection<PathMetadata> newChildren = new LinkedList<>();
+
+ for (PathMetadata child : oldChildren) {
+ FileStatus status = child.getFileStatus();
+ if (!expired(status, modTime, keyPrefix)) {
+ newChildren.add(child);
+ }
}
- }
- }
- }
+ if (newChildren.size() != oldChildren.size()) {
+ DirListingMetadata dlm =
+ new DirListingMetadata(path, newChildren, false);
+ localCache.put(path, new LocalMetadataEntry(dlm));
+ if (!path.isRoot()) {
+ DirListingMetadata parent = getDirListingMeta(path.getParent());
+ if (parent != null) {
+ parent.setAuthoritative(false);
+ }
+ }
+ }
+ });
}
private boolean expired(FileStatus status, long expiry, String keyPrefix) {
@@ -390,31 +403,26 @@ public class LocalMetadataStore implements MetadataStore {
}
@VisibleForTesting
- static <T> void deleteEntryByAncestor(Path ancestor, Cache<Path, T> cache,
- boolean tombstone) {
- for (Iterator<Map.Entry<Path, T>> it = cache.asMap().entrySet().iterator();
- it.hasNext();) {
- Map.Entry<Path, T> entry = it.next();
- Path f = entry.getKey();
- T meta = entry.getValue();
- if (isAncestorOf(ancestor, f)) {
- if (tombstone) {
- if (meta instanceof PathMetadata) {
- cache.put(f, (T) PathMetadata.tombstone(f));
- } else if (meta instanceof DirListingMetadata) {
- it.remove();
+ static void deleteEntryByAncestor(Path ancestor,
+ Cache<Path, LocalMetadataEntry> cache, boolean tombstone) {
+
+ cache.asMap().entrySet().stream()
+ .filter(entry -> isAncestorOf(ancestor, entry.getKey()))
+ .forEach(entry -> {
+ LocalMetadataEntry meta = entry.getValue();
+ Path path = entry.getKey();
+ if(meta.hasDirMeta()){
+ cache.invalidate(path);
+ } else if(tombstone && meta.hasPathMeta()){
+ meta.setPathMetadata(PathMetadata.tombstone(path));
} else {
- throw new IllegalStateException("Unknown type in cache");
+ cache.invalidate(path);
}
- } else {
- it.remove();
- }
- }
- }
+ });
}
/**
- * @return true iff 'ancestor' is ancestor dir in path 'f'.
+ * @return true if 'ancestor' is ancestor dir in path 'f'.
* All paths here are absolute. Dir does not count as its own ancestor.
*/
private static boolean isAncestorOf(Path ancestor, Path f) {
@@ -431,27 +439,41 @@ public class LocalMetadataStore implements MetadataStore {
* lock held.
*/
private void deleteCacheEntries(Path path, boolean tombstone) {
+ LocalMetadataEntry entry = localCache.getIfPresent(path);
+ // If there's no entry, delete should silently succeed
+ // (based on MetadataStoreTestBase#testDeleteNonExisting)
+ if(entry == null){
+ LOG.warn("Delete: path {} is missing from cache.", path);
+ return;
+ }
- // Remove target file/dir
+ // Remove target file entry
LOG.debug("delete file entry for {}", path);
- if (tombstone) {
- fileCache.put(path, PathMetadata.tombstone(path));
- } else {
- fileCache.invalidate(path);
+ if(entry.hasPathMeta()){
+ if (tombstone) {
+ PathMetadata pmd = PathMetadata.tombstone(path);
+ entry.setPathMetadata(pmd);
+ } else {
+ entry.setPathMetadata(null);
+ }
}
- // Update this and parent dir listing, if any
-
- /* If this path is a dir, remove its listing */
- LOG.debug("removing listing of {}", path);
+ // If this path is a dir, remove its listing
+ if(entry.hasDirMeta()) {
+ LOG.debug("removing listing of {}", path);
+ entry.setDirListingMetadata(null);
+ }
- dirCache.invalidate(path);
+ // If the entry is empty (contains no dirMeta or pathMeta) remove it from
+ // the cache.
+ if(!entry.hasDirMeta() && !entry.hasPathMeta()){
+ localCache.invalidate(entry);
+ }
/* Remove this path from parent's dir listing */
Path parent = path.getParent();
if (parent != null) {
- DirListingMetadata dir = null;
- dir = dirCache.getIfPresent(parent);
+ DirListingMetadata dir = getDirListingMeta(parent);
if (dir != null) {
LOG.debug("removing parent's entry for {} ", path);
if (tombstone) {
@@ -494,4 +516,23 @@ public class LocalMetadataStore implements MetadataStore {
public void updateParameters(Map<String, String> parameters)
throws IOException {
}
+
+ PathMetadata getFileMeta(Path p){
+ LocalMetadataEntry entry = localCache.getIfPresent(p);
+ if(entry != null && entry.hasPathMeta()){
+ return entry.getFileMeta();
+ } else {
+ return null;
+ }
+ }
+
+ DirListingMetadata getDirListingMeta(Path p){
+ LocalMetadataEntry entry = localCache.getIfPresent(p);
+ if(entry != null && entry.hasDirMeta()){
+ return entry.getDirListingMeta();
+ } else {
+ return null;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 56618cb..5a59400 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -836,7 +836,7 @@ public abstract class MetadataStoreTestBase extends HadoopTestBase {
throws IOException {
DirListingMetadata dirMeta = ms.listChildren(strToPath(pathStr));
if (!allowMissing()) {
- assertNotNull("Directory " + pathStr + " in cache", dirMeta);
+ assertNotNull("Directory " + pathStr + " is null in cache", dirMeta);
}
if (!allowMissing() || dirMeta != null) {
dirMeta = dirMeta.withoutTombstones();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c687a661/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index 074319f..2ea20b2 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.s3a.S3ATestUtils;
*/
public class TestLocalMetadataStore extends MetadataStoreTestBase {
- private static final String MAX_ENTRIES_STR = "16";
private final static class LocalMSContract extends AbstractMSContract {
@@ -48,7 +47,6 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
}
private LocalMSContract(Configuration config) throws IOException {
- config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
fs = FileSystem.getLocal(config);
}
@@ -76,8 +74,8 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
}
@Test
- public void testClearByAncestor() {
- Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder().build();
+ public void testClearByAncestor() throws Exception {
+ Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder().build();
// 1. Test paths without scheme/host
assertClearResult(cache, "", "/", 0);
@@ -122,7 +120,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
final long ttl = t1 + 50; // between t1 and t2
- Cache<Path, PathMetadata> cache = CacheBuilder.newBuilder()
+ Cache<Path, LocalMetadataEntry> cache = CacheBuilder.newBuilder()
.expireAfterWrite(ttl,
TimeUnit.NANOSECONDS /* nanos to avoid conversions */)
.ticker(testTicker)
@@ -143,7 +141,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
assertEquals("Cache should contain 3 records before eviction",
3, cache.size());
- PathMetadata pm1 = cache.getIfPresent(path1);
+ LocalMetadataEntry pm1 = cache.getIfPresent(path1);
assertNotNull("PathMetadata should not be null before eviction", pm1);
// set the ticker to a time when timed eviction should occur
@@ -159,7 +157,7 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
assertNull("PathMetadata should be null after eviction", pm1);
}
- private static void populateMap(Cache<Path, PathMetadata> cache,
+ private static void populateMap(Cache<Path, LocalMetadataEntry> cache,
String prefix) {
populateEntry(cache, new Path(prefix + "/dirA/dirB/"));
populateEntry(cache, new Path(prefix + "/dirA/dirB/dirC"));
@@ -168,23 +166,20 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
populateEntry(cache, new Path(prefix + "/dirA/file1"));
}
- private static void populateEntry(Cache<Path, PathMetadata> cache,
+ private static void populateEntry(Cache<Path, LocalMetadataEntry> cache,
Path path) {
- cache.put(path, new PathMetadata(new FileStatus(0, true, 0, 0, 0, path)));
+ FileStatus fileStatus = new FileStatus(0, true, 0, 0, 0, path);
+ cache.put(path, new LocalMetadataEntry(new PathMetadata(fileStatus)));
}
- private static int sizeOfMap(Cache<Path, PathMetadata> cache) {
- int count = 0;
- for (PathMetadata meta : cache.asMap().values()) {
- if (!meta.isDeleted()) {
- count++;
- }
- }
- return count;
+ private static long sizeOfMap(Cache<Path, LocalMetadataEntry> cache) {
+ return cache.asMap().values().stream()
+ .filter(entry -> !entry.getFileMeta().isDeleted())
+ .count();
}
- private static void assertClearResult(Cache<Path, PathMetadata> cache,
- String prefixStr, String pathStr, int leftoverSize) {
+ private static void assertClearResult(Cache<Path, LocalMetadataEntry> cache,
+ String prefixStr, String pathStr, int leftoverSize) throws IOException {
populateMap(cache, prefixStr);
LocalMetadataStore.deleteEntryByAncestor(new Path(prefixStr + pathStr),
cache, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-8379. Improve balancing
resources in already satisfied queues by using Capacity Scheduler preemption.
Contributed by Zian Chen.
Posted by gi...@apache.org.
YARN-8379. Improve balancing resources in already satisfied queues by using Capacity Scheduler preemption. Contributed by Zian Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29119430
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29119430
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29119430
Branch: refs/heads/HADOOP-15461
Commit: 291194302cc1a875d6d94ea93cf1184a3f1fc2cc
Parents: 384764c
Author: Sunil G <su...@apache.org>
Authored: Thu Jun 28 10:23:31 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Thu Jun 28 10:23:31 2018 -0700
----------------------------------------------------------------------
.../AbstractPreemptableResourceCalculator.java | 21 +-
.../CapacitySchedulerPreemptionContext.java | 2 +
.../CapacitySchedulerPreemptionUtils.java | 23 +-
.../capacity/FifoCandidatesSelector.java | 45 ++--
.../capacity/IntraQueueCandidatesSelector.java | 9 +-
.../capacity/PreemptableResourceCalculator.java | 7 +-
.../capacity/PreemptionCandidatesSelector.java | 11 +
.../ProportionalCapacityPreemptionPolicy.java | 129 +++++++---
...QueuePriorityContainerCandidateSelector.java | 16 +-
.../ReservedContainerCandidatesSelector.java | 16 +-
.../monitor/capacity/TempQueuePerPartition.java | 8 +-
.../CapacitySchedulerConfiguration.java | 17 ++
.../TestPreemptionForQueueWithPriorities.java | 58 +++++
...apacityPreemptionPolicyPreemptToBalance.java | 254 +++++++++++++++++++
...TestCapacitySchedulerSurgicalPreemption.java | 111 ++++++++
15 files changed, 637 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
index 64b3615..5b8360a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
@@ -42,6 +42,7 @@ public class AbstractPreemptableResourceCalculator {
protected final ResourceCalculator rc;
protected boolean isReservedPreemptionCandidatesSelector;
private Resource stepFactor;
+ private boolean allowQueuesBalanceAfterAllQueuesSatisfied;
static class TQComparator implements Comparator<TempQueuePerPartition> {
private ResourceCalculator rc;
@@ -83,15 +84,28 @@ public class AbstractPreemptableResourceCalculator {
* this will be set by different implementation of candidate
* selectors, please refer to TempQueuePerPartition#offer for
* details.
+ * @param allowQueuesBalanceAfterAllQueuesSatisfied
+ * Should resources be preempted from an over-served queue when the
+ * requesting queues are all at or over their guarantees?
+ * An example is, there're 10 queues under root, guaranteed resource
+ * of them are all 10%.
+ * Assume there're two queues are using resources, queueA uses 10%
+ * queueB uses 90%. For all queues are guaranteed, but it's not fair
+ * for queueA.
+ * We wanna make this behavior can be configured. By default it is
+ * not allowed.
+ *
*/
public AbstractPreemptableResourceCalculator(
CapacitySchedulerPreemptionContext preemptionContext,
- boolean isReservedPreemptionCandidatesSelector) {
+ boolean isReservedPreemptionCandidatesSelector,
+ boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
context = preemptionContext;
rc = preemptionContext.getResourceCalculator();
this.isReservedPreemptionCandidatesSelector =
isReservedPreemptionCandidatesSelector;
-
+ this.allowQueuesBalanceAfterAllQueuesSatisfied =
+ allowQueuesBalanceAfterAllQueuesSatisfied;
stepFactor = Resource.newInstance(0, 0);
for (ResourceInformation ri : stepFactor.getResources()) {
ri.setValue(1);
@@ -193,7 +207,8 @@ public class AbstractPreemptableResourceCalculator {
wQavail = Resources.componentwiseMin(wQavail, unassigned);
Resource wQidle = sub.offer(wQavail, rc, totGuarant,
- isReservedPreemptionCandidatesSelector);
+ isReservedPreemptionCandidatesSelector,
+ allowQueuesBalanceAfterAllQueuesSatisfied);
Resource wQdone = Resources.subtract(wQavail, wQidle);
if (Resources.greaterThan(rc, totGuarant, wQdone, Resources.none())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
index 098acdd..7985296 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionContext.java
@@ -70,6 +70,8 @@ public interface CapacitySchedulerPreemptionContext {
float getMaxAllowableLimitForIntraQueuePreemption();
+ long getDefaultMaximumKillWaitTimeout();
+
@Unstable
IntraQueuePreemptionOrderPolicy getIntraQueuePreemptionOrderPolicy();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
index 690eb02..ed50eff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
@@ -151,6 +151,7 @@ public class CapacitySchedulerPreemptionUtils {
Map<String, Resource> resourceToObtainByPartitions,
RMContainer rmContainer, Resource clusterResource,
Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
Resource totalPreemptionAllowed, boolean conservativeDRF) {
ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId();
@@ -218,7 +219,7 @@ public class CapacitySchedulerPreemptionUtils {
}
// Add to preemptMap
- addToPreemptMap(preemptMap, attemptId, rmContainer);
+ addToPreemptMap(preemptMap, curCandidates, attemptId, rmContainer);
return true;
}
@@ -230,15 +231,23 @@ public class CapacitySchedulerPreemptionUtils {
return context.getScheduler().getSchedulerNode(nodeId).getPartition();
}
- private static void addToPreemptMap(
+ protected static void addToPreemptMap(
Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
ApplicationAttemptId appAttemptId, RMContainer containerToPreempt) {
- Set<RMContainer> set = preemptMap.get(appAttemptId);
- if (null == set) {
- set = new HashSet<>();
- preemptMap.put(appAttemptId, set);
+ Set<RMContainer> setForToPreempt = preemptMap.get(appAttemptId);
+ Set<RMContainer> setForCurCandidates = curCandidates.get(appAttemptId);
+ if (null == setForToPreempt) {
+ setForToPreempt = new HashSet<>();
+ preemptMap.put(appAttemptId, setForToPreempt);
}
- set.add(containerToPreempt);
+ setForToPreempt.add(containerToPreempt);
+
+ if (null == setForCurCandidates) {
+ setForCurCandidates = new HashSet<>();
+ curCandidates.put(appAttemptId, setForCurCandidates);
+ }
+ setForCurCandidates.add(containerToPreempt);
}
private static boolean preemptMapContains(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index 3b2fcbb..c2735f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -42,19 +43,25 @@ public class FifoCandidatesSelector
private static final Log LOG =
LogFactory.getLog(FifoCandidatesSelector.class);
private PreemptableResourceCalculator preemptableAmountCalculator;
+ private boolean allowQueuesBalanceAfterAllQueuesSatisfied;
FifoCandidatesSelector(CapacitySchedulerPreemptionContext preemptionContext,
- boolean includeReservedResource) {
+ boolean includeReservedResource,
+ boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
super(preemptionContext);
+ this.allowQueuesBalanceAfterAllQueuesSatisfied =
+ allowQueuesBalanceAfterAllQueuesSatisfied;
preemptableAmountCalculator = new PreemptableResourceCalculator(
- preemptionContext, includeReservedResource);
+ preemptionContext, includeReservedResource,
+ allowQueuesBalanceAfterAllQueuesSatisfied);
}
@Override
public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
Resource clusterResource, Resource totalPreemptionAllowed) {
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
// Calculate how much resources we need to preempt
preemptableAmountCalculator.computeIdealAllocation(clusterResource,
totalPreemptionAllowed);
@@ -110,7 +117,7 @@ public class FifoCandidatesSelector
boolean preempted = CapacitySchedulerPreemptionUtils
.tryPreemptContainerAndDeductResToObtain(rc,
preemptionContext, resToObtainByPartition, c,
- clusterResource, selectedCandidates,
+ clusterResource, selectedCandidates, curCandidates,
totalPreemptionAllowed, false);
if (!preempted) {
continue;
@@ -134,7 +141,7 @@ public class FifoCandidatesSelector
preemptFrom(fc, clusterResource, resToObtainByPartition,
skippedAMContainerlist, skippedAMSize, selectedCandidates,
- totalPreemptionAllowed);
+ curCandidates, totalPreemptionAllowed);
}
// Can try preempting AMContainers (still saving atmost
@@ -145,15 +152,15 @@ public class FifoCandidatesSelector
leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
leafQueue.getMaxAMResourcePerQueuePercent());
- preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
- resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
- totalPreemptionAllowed);
+ preemptAMContainers(clusterResource, selectedCandidates, curCandidates,
+ skippedAMContainerlist, resToObtainByPartition, skippedAMSize,
+ maxAMCapacityForThisQueue, totalPreemptionAllowed);
} finally {
leafQueue.getReadLock().unlock();
}
}
- return selectedCandidates;
+ return curCandidates;
}
/**
@@ -169,6 +176,7 @@ public class FifoCandidatesSelector
*/
private void preemptAMContainers(Resource clusterResource,
Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
List<RMContainer> skippedAMContainerlist,
Map<String, Resource> resToObtainByPartition, Resource skippedAMSize,
Resource maxAMCapacityForThisQueue, Resource totalPreemptionAllowed) {
@@ -187,7 +195,7 @@ public class FifoCandidatesSelector
boolean preempted = CapacitySchedulerPreemptionUtils
.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
resToObtainByPartition, c, clusterResource, preemptMap,
- totalPreemptionAllowed, false);
+ curCandidates, totalPreemptionAllowed, false);
if (preempted) {
Resources.subtractFrom(skippedAMSize, c.getAllocatedResource());
}
@@ -203,6 +211,7 @@ public class FifoCandidatesSelector
Resource clusterResource, Map<String, Resource> resToObtainByPartition,
List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
Map<ApplicationAttemptId, Set<RMContainer>> selectedContainers,
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
Resource totalPreemptionAllowed) {
ApplicationAttemptId appId = app.getApplicationAttemptId();
@@ -219,9 +228,10 @@ public class FifoCandidatesSelector
}
// Try to preempt this container
- CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
- rc, preemptionContext, resToObtainByPartition, c, clusterResource,
- selectedContainers, totalPreemptionAllowed, false);
+ CapacitySchedulerPreemptionUtils
+ .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
+ resToObtainByPartition, c, clusterResource, selectedContainers,
+ curCandidates, totalPreemptionAllowed, false);
if (!preemptionContext.isObserveOnly()) {
preemptionContext.getRMContext().getDispatcher().getEventHandler()
@@ -262,9 +272,14 @@ public class FifoCandidatesSelector
}
// Try to preempt this container
- CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
- rc, preemptionContext, resToObtainByPartition, c, clusterResource,
- selectedContainers, totalPreemptionAllowed, false);
+ CapacitySchedulerPreemptionUtils
+ .tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
+ resToObtainByPartition, c, clusterResource, selectedContainers,
+ curCandidates, totalPreemptionAllowed, false);
}
}
+
+ public boolean getAllowQueuesBalanceAfterAllQueuesSatisfied() {
+ return allowQueuesBalanceAfterAllQueuesSatisfied;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
index 8ab9507..c52fd95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
@@ -122,7 +122,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
Resource clusterResource, Resource totalPreemptedResourceAllowed) {
-
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
// 1. Calculate the abnormality within each queue one by one.
computeIntraQueuePreemptionDemand(
clusterResource, totalPreemptedResourceAllowed, selectedCandidates);
@@ -182,7 +182,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
leafQueue.getReadLock().lock();
for (FiCaSchedulerApp app : apps) {
preemptFromLeastStarvedApp(leafQueue, app, selectedCandidates,
- clusterResource, totalPreemptedResourceAllowed,
+ curCandidates, clusterResource, totalPreemptedResourceAllowed,
resToObtainByPartition, rollingResourceUsagePerUser);
}
} finally {
@@ -191,7 +191,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
}
}
- return selectedCandidates;
+ return curCandidates;
}
private void initializeUsageAndUserLimitForCompute(Resource clusterResource,
@@ -211,6 +211,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
private void preemptFromLeastStarvedApp(LeafQueue leafQueue,
FiCaSchedulerApp app,
Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates,
Resource clusterResource, Resource totalPreemptedResourceAllowed,
Map<String, Resource> resToObtainByPartition,
Map<String, Resource> rollingResourceUsagePerUser) {
@@ -270,7 +271,7 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector {
boolean ret = CapacitySchedulerPreemptionUtils
.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
resToObtainByPartition, c, clusterResource, selectedCandidates,
- totalPreemptedResourceAllowed, true);
+ curCandidates, totalPreemptedResourceAllowed, true);
// Subtract from respective user's resource usage once a container is
// selected for preemption.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
index 08d834e..89a015e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
@@ -48,11 +48,14 @@ public class PreemptableResourceCalculator
* @param isReservedPreemptionCandidatesSelector this will be set by
* different implementation of candidate selectors, please refer to
* TempQueuePerPartition#offer for details.
+ * @param allowQueuesBalanceAfterAllQueuesSatisfied
*/
public PreemptableResourceCalculator(
CapacitySchedulerPreemptionContext preemptionContext,
- boolean isReservedPreemptionCandidatesSelector) {
- super(preemptionContext, isReservedPreemptionCandidatesSelector);
+ boolean isReservedPreemptionCandidatesSelector,
+ boolean allowQueuesBalanceAfterAllQueuesSatisfied) {
+ super(preemptionContext, isReservedPreemptionCandidatesSelector,
+ allowQueuesBalanceAfterAllQueuesSatisfied);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
index 4d8afaf..3c97364 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptionCandidatesSelector.java
@@ -34,6 +34,7 @@ import java.util.Set;
public abstract class PreemptionCandidatesSelector {
protected CapacitySchedulerPreemptionContext preemptionContext;
protected ResourceCalculator rc;
+ private long maximumKillWaitTime = -1;
PreemptionCandidatesSelector(
CapacitySchedulerPreemptionContext preemptionContext) {
@@ -77,4 +78,14 @@ public abstract class PreemptionCandidatesSelector {
});
}
+ public long getMaximumKillWaitTimeMs() {
+ if (maximumKillWaitTime > 0) {
+ return maximumKillWaitTime;
+ }
+ return preemptionContext.getDefaultMaximumKillWaitTimeout();
+ }
+
+ public void setMaximumKillWaitTime(long maximumKillWaitTime) {
+ this.maximumKillWaitTime = maximumKillWaitTime;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index cc69fba..036fd2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -131,6 +131,8 @@ public class ProportionalCapacityPreemptionPolicy
private List<PreemptionCandidatesSelector> candidatesSelectionPolicies;
private Set<String> allPartitions;
private Set<String> leafQueueNames;
+ Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> pcsMap;
// Preemptable Entities, synced from scheduler at every run
private Map<String, PreemptableQueue> preemptableQueues;
@@ -249,7 +251,21 @@ public class ProportionalCapacityPreemptionPolicy
// initialize candidates preemption selection policies
candidatesSelectionPolicies.add(new FifoCandidatesSelector(this,
- additionalPreemptionBasedOnReservedResource));
+ additionalPreemptionBasedOnReservedResource, false));
+
+ // Do we need to do preemption to balance queue even after queues get satisfied?
+ boolean isPreemptionToBalanceRequired = config.getBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED);
+ long maximumKillWaitTimeForPreemptionToQueueBalance = config.getLong(
+ CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+ CapacitySchedulerConfiguration.DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION);
+ if (isPreemptionToBalanceRequired) {
+ PreemptionCandidatesSelector selector = new FifoCandidatesSelector(this,
+ false, true);
+ selector.setMaximumKillWaitTime(maximumKillWaitTimeForPreemptionToQueueBalance);
+ candidatesSelectionPolicies.add(selector);
+ }
// Do we need to specially consider intra queue
boolean isIntraQueuePreemptionEnabled = config.getBoolean(
@@ -282,7 +298,8 @@ public class ProportionalCapacityPreemptionPolicy
"select_based_on_reserved_containers = " +
selectCandidatesForResevedContainers + "\n" +
"additional_res_balance_based_on_reserved_containers = " +
- additionalPreemptionBasedOnReservedResource);
+ additionalPreemptionBasedOnReservedResource + "\n" +
+ "Preemption-to-balance-queue-enabled = " + isPreemptionToBalanceRequired);
csConfig = config;
}
@@ -308,44 +325,60 @@ public class ProportionalCapacityPreemptionPolicy
}
private void preemptOrkillSelectedContainerAfterWait(
- Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
- long currentTime) {
+ Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> toPreemptPerSelector, long currentTime) {
+ int toPreemptCount = 0;
+ for (Map<ApplicationAttemptId, Set<RMContainer>> containers :
+ toPreemptPerSelector.values()) {
+ toPreemptCount += containers.size();
+ }
if (LOG.isDebugEnabled()) {
LOG.debug(
"Starting to preempt containers for selectedCandidates and size:"
- + selectedCandidates.size());
+ + toPreemptCount);
}
// preempt (or kill) the selected containers
- for (Map.Entry<ApplicationAttemptId, Set<RMContainer>> e : selectedCandidates
+ // We need toPreemptPerSelector here to match list of containers to
+ // its selector so that we can get custom timeout per selector when
+ // checking if current container should be killed or not
+ for (Map.Entry<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> pc : toPreemptPerSelector
.entrySet()) {
- ApplicationAttemptId appAttemptId = e.getKey();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Send to scheduler: in app=" + appAttemptId
- + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
- }
- for (RMContainer container : e.getValue()) {
- // if we tried to preempt this for more than maxWaitTime
- if (preemptionCandidates.get(container) != null
- && preemptionCandidates.get(container)
- + maxWaitTime <= currentTime) {
- // kill it
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(appAttemptId, container,
- SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
- preemptionCandidates.remove(container);
- } else {
- if (preemptionCandidates.get(container) != null) {
- // We already updated the information to scheduler earlier, we need
- // not have to raise another event.
- continue;
+ Map<ApplicationAttemptId, Set<RMContainer>> cMap = pc.getValue();
+ if (cMap.size() > 0) {
+ for (Map.Entry<ApplicationAttemptId,
+ Set<RMContainer>> e : cMap.entrySet()) {
+ ApplicationAttemptId appAttemptId = e.getKey();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Send to scheduler: in app=" + appAttemptId
+ + " #containers-to-be-preemptionCandidates=" + e.getValue().size());
+ }
+ for (RMContainer container : e.getValue()) {
+ // if we tried to preempt this for more than maxWaitTime, this
+ // should be based on custom timeout per container per selector
+ if (preemptionCandidates.get(container) != null
+ && preemptionCandidates.get(container)
+ + pc.getKey().getMaximumKillWaitTimeMs() <= currentTime) {
+ // kill it
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE));
+ preemptionCandidates.remove(container);
+ } else {
+ if (preemptionCandidates.get(container) != null) {
+ // We already updated the information to scheduler earlier, we need
+ // not have to raise another event.
+ continue;
+ }
+
+ //otherwise just send preemption events
+ rmContext.getDispatcher().getEventHandler().handle(
+ new ContainerPreemptEvent(appAttemptId, container,
+ SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
+ preemptionCandidates.put(container, currentTime);
+ }
}
-
- //otherwise just send preemption events
- rmContext.getDispatcher().getEventHandler().handle(
- new ContainerPreemptEvent(appAttemptId, container,
- SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION));
- preemptionCandidates.put(container, currentTime);
}
}
}
@@ -438,6 +471,8 @@ public class ProportionalCapacityPreemptionPolicy
// queue and each application
Map<ApplicationAttemptId, Set<RMContainer>> toPreempt =
new HashMap<>();
+ Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> toPreemptPerSelector = new HashMap<>();;
for (PreemptionCandidatesSelector selector :
candidatesSelectionPolicies) {
long startTime = 0;
@@ -447,20 +482,27 @@ public class ProportionalCapacityPreemptionPolicy
selector.getClass().getName()));
startTime = clock.getTime();
}
- toPreempt = selector.selectCandidates(toPreempt,
- clusterResources, totalPreemptionAllowed);
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates =
+ selector.selectCandidates(toPreempt, clusterResources,
+ totalPreemptionAllowed);
+ toPreemptPerSelector.putIfAbsent(selector, curCandidates);
if (LOG.isDebugEnabled()) {
LOG.debug(MessageFormat
.format("{0} uses {1} millisecond to run",
selector.getClass().getName(), clock.getTime() - startTime));
int totalSelected = 0;
+ int curSelected = 0;
for (Set<RMContainer> set : toPreempt.values()) {
totalSelected += set.size();
}
+ for (Set<RMContainer> set : curCandidates.values()) {
+ curSelected += set.size();
+ }
LOG.debug(MessageFormat
- .format("So far, total {0} containers selected to be preempted",
- totalSelected));
+ .format("So far, total {0} containers selected to be preempted, {1}"
+ + " containers selected this round\n",
+ totalSelected, curSelected));
}
}
@@ -483,8 +525,10 @@ public class ProportionalCapacityPreemptionPolicy
long currentTime = clock.getTime();
+ pcsMap = toPreemptPerSelector;
+
// preempt (or kill) the selected containers
- preemptOrkillSelectedContainerAfterWait(toPreempt, currentTime);
+ preemptOrkillSelectedContainerAfterWait(toPreemptPerSelector, currentTime);
// cleanup staled preemption candidates
cleanupStaledPreemptionCandidates(currentTime);
@@ -689,6 +733,12 @@ public class ProportionalCapacityPreemptionPolicy
return queueToPartitions;
}
+ @VisibleForTesting
+ Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> getToPreemptCandidatesPerSelector() {
+ return pcsMap;
+ }
+
@Override
public int getClusterMaxApplicationPriority() {
return scheduler.getMaxClusterLevelAppPriority().getPriority();
@@ -730,4 +780,9 @@ public class ProportionalCapacityPreemptionPolicy
public IntraQueuePreemptionOrderPolicy getIntraQueuePreemptionOrderPolicy() {
return intraQueuePreemptionOrderPolicy;
}
+
+ @Override
+ public long getDefaultMaximumKillWaitTimeout() {
+ return maxWaitTime;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
index 4a169af..78a9988 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/QueuePriorityContainerCandidateSelector.java
@@ -380,6 +380,7 @@ public class QueuePriorityContainerCandidateSelector
Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
Resource clusterResource,
Resource totalPreemptedResourceAllowed) {
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
// Initialize digraph from queues
// TODO (wangda): only do this when queue refreshed.
priorityDigraph.clear();
@@ -388,7 +389,7 @@ public class QueuePriorityContainerCandidateSelector
// When all queues are set to same priority, or priority is not respected,
// direct return.
if (priorityDigraph.isEmpty()) {
- return selectedCandidates;
+ return curCandidates;
}
// Save parameters to be shared by other methods
@@ -478,13 +479,9 @@ public class QueuePriorityContainerCandidateSelector
.getReservedResource());
}
- Set<RMContainer> containers = selectedCandidates.get(
- c.getApplicationAttemptId());
- if (null == containers) {
- containers = new HashSet<>();
- selectedCandidates.put(c.getApplicationAttemptId(), containers);
- }
- containers.add(c);
+ // Add to preemptMap
+ CapacitySchedulerPreemptionUtils.addToPreemptMap(selectedCandidates,
+ curCandidates, c.getApplicationAttemptId(), c);
// Update totalPreemptionResourceAllowed
Resources.subtractFrom(totalPreemptedResourceAllowed,
@@ -504,7 +501,6 @@ public class QueuePriorityContainerCandidateSelector
}
}
}
-
- return selectedCandidates;
+ return curCandidates;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
index ff100d9..bdb7e8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ReservedContainerCandidatesSelector.java
@@ -31,7 +31,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -63,7 +62,7 @@ public class ReservedContainerCandidatesSelector
CapacitySchedulerPreemptionContext preemptionContext) {
super(preemptionContext);
preemptableAmountCalculator = new PreemptableResourceCalculator(
- preemptionContext, true);
+ preemptionContext, true, false);
}
@Override
@@ -71,6 +70,7 @@ public class ReservedContainerCandidatesSelector
Map<ApplicationAttemptId, Set<RMContainer>> selectedCandidates,
Resource clusterResource,
Resource totalPreemptedResourceAllowed) {
+ Map<ApplicationAttemptId, Set<RMContainer>> curCandidates = new HashMap<>();
// Calculate how much resources we need to preempt
preemptableAmountCalculator.computeIdealAllocation(clusterResource,
totalPreemptedResourceAllowed);
@@ -101,14 +101,10 @@ public class ReservedContainerCandidatesSelector
selectedCandidates, totalPreemptedResourceAllowed, false);
if (null != preemptionResult) {
for (RMContainer c : preemptionResult.selectedContainers) {
- ApplicationAttemptId appId = c.getApplicationAttemptId();
- Set<RMContainer> containers = selectedCandidates.get(appId);
- if (null == containers) {
- containers = new HashSet<>();
- selectedCandidates.put(appId, containers);
- }
+ // Add to preemptMap
+ CapacitySchedulerPreemptionUtils.addToPreemptMap(selectedCandidates,
+ curCandidates, c.getApplicationAttemptId(), c);
- containers.add(c);
if (LOG.isDebugEnabled()) {
LOG.debug(this.getClass().getName() + " Marked container=" + c
.getContainerId() + " from queue=" + c.getQueueName()
@@ -118,7 +114,7 @@ public class ReservedContainerCandidatesSelector
}
}
- return selectedCandidates;
+ return curCandidates;
}
private Resource getPreemptableResource(String queueName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 4214acc..4fb1862 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -138,7 +138,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
// This function "accepts" all the resources it can (pending) and return
// the unused ones
Resource offer(Resource avail, ResourceCalculator rc,
- Resource clusterResource, boolean considersReservedResource) {
+ Resource clusterResource, boolean considersReservedResource,
+ boolean allowQueueBalanceAfterAllSafisfied) {
Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
Resources.subtract(getMax(), idealAssigned),
Resource.newInstance(0, 0));
@@ -179,7 +180,10 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
// leaf queues. Such under-utilized leaf queue could preemption resources
// from over-utilized leaf queue located at other hierarchies.
- accepted = filterByMaxDeductAssigned(rc, clusterResource, accepted);
+ // Allow queues can continue grow and balance even if all queues are satisfied.
+ if (!allowQueueBalanceAfterAllSafisfied) {
+ accepted = filterByMaxDeductAssigned(rc, clusterResource, accepted);
+ }
// accepted so far contains the "quota acceptable" amount, we now filter by
// locality acceptable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 76eaac0..f94654e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1460,6 +1460,23 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String DEFAULT_INTRAQUEUE_PREEMPTION_ORDER_POLICY = "userlimit_first";
/**
+ * Should we allow queues continue grow after all queue reaches their
+ * guaranteed capacity.
+ */
+ public static final String PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED =
+ PREEMPTION_CONFIG_PREFIX + "preemption-to-balance-queue-after-satisfied.enabled";
+ public static final boolean DEFAULT_PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED = false;
+
+ /**
+ * How long we will wait to balance queues, by default it is 5 mins.
+ */
+ public static final String MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION =
+ PREEMPTION_CONFIG_PREFIX + "preemption-to-balance-queue-after-satisfied.max-wait-before-kill";
+ public static final long
+ DEFAULT_MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION =
+ 300 * 1000;
+
+ /**
* Maximum application for a queue to be used when application per queue is
* not defined.To be consistent with previous version the default value is set
* as UNDEFINED.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
index 6a953cf..38c2a2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -538,4 +539,61 @@ public class TestPreemptionForQueueWithPriorities
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
+
+ @Test
+ public void testPriorityPreemptionForBalanceBetweenSatisfiedQueues()
+ throws IOException {
+ /**
+ * All queues are beyond guarantee, c has higher priority than b.
+ * c ask for more resource, and there is no idle left, c should preempt
+ * some resource from b but won’t let b under its guarantee.
+ *
+ * Queue structure is:
+ *
+ * <pre>
+ * root
+ * / | \
+ * a b c
+ * </pre>
+ *
+ * For priorities
+ * - a=1
+ * - b=1
+ * - c=2
+ *
+ */
+ String labelsConfig = "=100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100 100 100 100]);" + //root
+ "-a(=[30 100 0 0]){priority=1};" + // a
+ "-b(=[30 100 40 50]){priority=1};" + // b
+ "-c(=[40 100 60 25]){priority=2}"; // c
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "b\t(1,1,n1,,40,false);" + // app1 in b
+ "c\t(1,1,n1,,60,false)"; // app2 in c
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration(conf);
+ boolean isPreemptionToBalanceRequired = true;
+ newConf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ isPreemptionToBalanceRequired);
+ when(cs.getConfiguration()).thenReturn(newConf);
+ policy.editSchedule();
+
+ // IdealAssigned b: 30 c: 70. initIdealAssigned: b: 30 c: 40, even though
+ // b and c has same relativeAssigned=1.0f(idealAssigned / guaranteed),
+ // since c has higher priority, c will be put in mostUnderServedQueue and
+ // get all remain 30 capacity.
+ verify(mDisp, times(10)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(1))));
+ verify(mDisp, never()).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(2))));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
new file mode 100644
index 0000000..22e8f63
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyPreemptToBalance.java
@@ -0,0 +1,254 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TestProportionalCapacityPreemptionPolicyPreemptToBalance
+ extends ProportionalCapacityPreemptionPolicyMockFramework {
+
+ @Test
+ public void testPreemptionToBalanceDisabled() throws IOException {
+ String labelsConfig = "=100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100 100 100 100]);" + //root
+ "-a(=[30 100 10 30]);" + // a
+ "-b(=[30 100 40 30]);" + // b
+ "-c(=[30 100 50 30]);" + // c
+ "-d(=[10 100 0 0])"; // d
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a\t(1,1,n1,,10,false);" + // app1 in a
+ "b\t(1,1,n1,,40,false);" + // app2 in b
+ "c\t(1,1,n1,,50,false)"; // app3 in c
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ // I_A: A:30 B:35 C:35, preempt 5 from B and 15 from C to A
+ verify(mDisp, times(5)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(2))));
+ verify(mDisp, times(15)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(3))));
+
+ assertEquals(30, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(35, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(35, policy.getQueuePartitions().get("c")
+ .get("").getIdealAssigned().getMemorySize());
+ }
+
+ @Test
+ public void testPreemptionToBalanceEnabled() throws IOException {
+ String labelsConfig = "=100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100 100 100 100]);" + //root
+ "-a(=[30 100 10 30]);" + // a
+ "-b(=[30 100 40 30]);" + // b
+ "-c(=[30 100 50 30]);" + // c
+ "-d(=[10 100 0 0])"; // d
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a\t(1,1,n1,,10,false);" + // app1 in a
+ "b\t(1,1,n1,,40,false);" + // app2 in b
+ "c\t(1,1,n1,,50,false)"; // app3 in c
+
+ // enable preempt to balance and ideal assignment will change.
+ boolean isPreemptionToBalanceEnabled = true;
+ conf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ isPreemptionToBalanceEnabled);
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ // I_A: A:33 B:33 C:33, preempt 7 from B and 17 from C to A
+ verify(mDisp, times(7)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(2))));
+ verify(mDisp, times(17)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(3))));
+
+ assertEquals(33, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(33, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(33, policy.getQueuePartitions().get("c")
+ .get("").getIdealAssigned().getMemorySize());
+ }
+
+
+ @Test
+ public void testPreemptionToBalanceUsedPlusPendingLessThanGuaranteed()
+ throws IOException{
+ String labelsConfig = "=100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100 100 100 100]);" + //root
+ "-a(=[30 100 10 6]);" + // a
+ "-b(=[30 100 40 30]);" + // b
+ "-c(=[30 100 50 30]);" + // c
+ "-d(=[10 100 0 0])"; // d
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a\t(1,1,n1,,10,false);" + // app1 in a
+ "b\t(1,1,n1,,40,false);" + // app2 in b
+ "c\t(1,1,n1,,50,false)"; // app3 in c
+
+ boolean isPreemptionToBalanceEnabled = true;
+ conf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ isPreemptionToBalanceEnabled);
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ // I_A: A:15 B:42 C:43, preempt 7 from B and 17 from C to A
+ verify(mDisp, times(8)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(3))));
+
+ assertEquals(16, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(42, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(42, policy.getQueuePartitions().get("c")
+ .get("").getIdealAssigned().getMemorySize());
+ }
+
+ @Test
+ public void testPreemptionToBalanceWithVcoreResource() throws IOException {
+ Logger.getRootLogger().setLevel(Level.DEBUG);
+ String labelsConfig = "=100:100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100:100 100:100 100:100 120:140]);" + //root
+ "-a(=[60:60 100:100 40:40 70:40]);" + // a
+ "-b(=[40:40 100:100 60:60 50:100])"; // b
+
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a\t(1,1:1,n1,,40,false);" + // app1 in a
+ "b\t(1,1:1,n1,,60,false)"; // app2 in b
+
+ boolean isPreemptionToBalanceEnabled = true;
+ conf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ isPreemptionToBalanceEnabled);
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true);
+ policy.editSchedule();
+
+ // 21 containers will be preempted here
+ verify(mDisp, times(21)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.
+ IsPreemptionRequestFor(getAppAttemptId(2))));
+
+ assertEquals(60, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(60, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getVirtualCores());
+ assertEquals(40, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(40, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getVirtualCores());
+ }
+
+ @Test
+ public void testPreemptionToBalanceWithConfiguredTimeout() throws IOException {
+ Logger.getRootLogger().setLevel(Level.DEBUG);
+ String labelsConfig = "=100:100,true"; // default partition
+ String nodesConfig = "n1="; // only one node
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100:100 100:100 100:100 120:140]);" + //root
+ "-a(=[60:60 100:100 40:40 70:40]);" + // a
+ "-b(=[40:40 100:100 60:60 50:100])"; // b
+
+ String appsConfig =
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a\t(1,1:1,n1,,40,false);" + // app1 in a
+ "b\t(1,1:1,n1,,60,false)"; // app2 in b
+
+ boolean isPreemptionToBalanceEnabled = true;
+ conf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ isPreemptionToBalanceEnabled);
+ final long FB_MAX_BEFORE_KILL = 60 *1000;
+ conf.setLong(
+ CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+ FB_MAX_BEFORE_KILL);
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true);
+ policy.editSchedule();
+
+ Map<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> pcps= policy.getToPreemptCandidatesPerSelector();
+
+ String FIFO_CANDIDATE_SELECTOR = "FifoCandidatesSelector";
+ boolean hasFifoSelector = false;
+ for (Map.Entry<PreemptionCandidatesSelector, Map<ApplicationAttemptId,
+ Set<RMContainer>>> pc : pcps.entrySet()) {
+ if (pc.getKey().getClass().getSimpleName().equals(FIFO_CANDIDATE_SELECTOR)) {
+ FifoCandidatesSelector pcs = (FifoCandidatesSelector) pc.getKey();
+ if (pcs.getAllowQueuesBalanceAfterAllQueuesSatisfied() == true) {
+ hasFifoSelector = true;
+ assertEquals(pcs.getMaximumKillWaitTimeMs(), FB_MAX_BEFORE_KILL);
+ }
+ }
+ }
+
+ assertEquals(hasFifoSelector, true);
+
+ // 21 containers will be preempted here
+ verify(mDisp, times(21)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.
+ IsPreemptionRequestFor(getAppAttemptId(2))));
+
+ assertEquals(60, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(60, policy.getQueuePartitions().get("a")
+ .get("").getIdealAssigned().getVirtualCores());
+ assertEquals(40, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getMemorySize());
+ assertEquals(40, policy.getQueuePartitions().get("b")
+ .get("").getIdealAssigned().getVirtualCores());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29119430/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 2aff82d..800789a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -1111,5 +1111,116 @@ public class TestCapacitySchedulerSurgicalPreemption
rm1.close();
}
+ @Test(timeout = 600000)
+ public void testPreemptionToBalanceWithCustomTimeout() throws Exception {
+ /**
+ * Test case: Submit two application (app1/app2) to different queues, queue
+ * structure:
+ *
+ * <pre>
+ * Root
+ * / | \
+ * a b c
+ * 10 20 70
+ * </pre>
+ *
+ * 1) Two nodes (n1/n2) in the cluster, each of them has 20G.
+ *
+ * 2) app1 submit to queue-b, asks for 1G * 5
+ *
+ * 3) app2 submit to queue-c, ask for one 4G container (for AM)
+ *
+ * After preemption, we should expect:
+ * 1. Preempt 4 containers from app1
+ * 2. the selected containers will be killed after configured timeout.
+ * 3. AM of app2 successfully allocated.
+ */
+ conf.setBoolean(
+ CapacitySchedulerConfiguration.PREEMPTION_TO_BALANCE_QUEUES_BEYOND_GUARANTEED,
+ true);
+ conf.setLong(
+ CapacitySchedulerConfiguration.MAX_WAIT_BEFORE_KILL_FOR_QUEUE_BALANCE_PREEMPTION,
+ 20*1000);
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(
+ this.conf);
+
+ MockRM rm1 = new MockRM(conf);
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+
+ MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 20 * GB);
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+ // launch an app to queue, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ am1.allocate("*", 1 * GB, 38, new ArrayList<ContainerId>());
+
+ // Do allocation for node1/node2
+ for (int i = 0; i < 38; i++) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ }
+
+ // App1 should have 39 containers now
+ FiCaSchedulerApp schedulerApp1 = cs.getApplicationAttempt(
+ am1.getApplicationAttemptId());
+ Assert.assertEquals(39, schedulerApp1.getLiveContainers().size());
+ // 20 from n1 and 19 from n2
+ waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode1.getNodeID()),
+ am1.getApplicationAttemptId(), 20);
+ waitNumberOfLiveContainersOnNodeFromApp(cs.getNode(rmNode2.getNodeID()),
+ am1.getApplicationAttemptId(), 19);
+
+
+ // Submit app2 to queue-c and asks for a 4G container for AM
+ RMApp app2 = rm1.submitApp(4 * GB, "app", "user", null, "c");
+ FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt(
+ ApplicationAttemptId.newInstance(app2.getApplicationId(), 1));
+
+ // Call editSchedule: containers are selected to be preemption candidate
+ SchedulingMonitorManager smm = ((CapacityScheduler) rm1.
+ getResourceScheduler()).getSchedulingMonitorManager();
+ SchedulingMonitor smon = smm.getAvailableSchedulingMonitor();
+ ProportionalCapacityPreemptionPolicy editPolicy =
+ (ProportionalCapacityPreemptionPolicy) smon.getSchedulingEditPolicy();
+ editPolicy.editSchedule();
+ Assert.assertEquals(4, editPolicy.getToPreemptContainers().size());
+
+ // check live containers immediately, nothing happen
+ Assert.assertEquals(39, schedulerApp1.getLiveContainers().size());
+
+ Thread.sleep(20*1000);
+ // Call editSchedule again: selected containers are killed
+ editPolicy.editSchedule();
+ waitNumberOfLiveContainersFromApp(schedulerApp1, 35);
+
+ // Call allocation, containers are reserved
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ waitNumberOfReservedContainersFromApp(schedulerApp2, 1);
+
+ // Call editSchedule twice and allocation once, container should get allocated
+ editPolicy.editSchedule();
+ editPolicy.editSchedule();
+
+ int tick = 0;
+ while (schedulerApp2.getLiveContainers().size() != 1 && tick < 10) {
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ tick++;
+ Thread.sleep(100);
+ }
+ waitNumberOfReservedContainersFromApp(schedulerApp2, 0);
+
+ rm1.close();
+
+
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-8481. AMRMProxyPolicies
should accept heartbeat response from new/unknown subclusters. Contributed by
Botong Huang.
Posted by gi...@apache.org.
YARN-8481. AMRMProxyPolicies should accept heartbeat response from new/unknown subclusters. Contributed by Botong Huang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdb08442
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdb08442
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdb08442
Branch: refs/heads/HADOOP-15461
Commit: cdb084426bc27a9f902da0b6927a3354a307dd82
Parents: d36f6b9
Author: Giovanni Matteo Fumarola <gi...@apache.com>
Authored: Fri Jun 29 11:47:30 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.com>
Committed: Fri Jun 29 11:47:30 2018 -0700
----------------------------------------------------------------------
.../policies/amrmproxy/BroadcastAMRMProxyPolicy.java | 11 -----------
.../policies/amrmproxy/RejectAMRMProxyPolicy.java | 4 ----
.../TestBroadcastAMRMProxyFederationPolicy.java | 11 +++--------
3 files changed, 3 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
index 679f4d5..7fddb8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/BroadcastAMRMProxyPolicy.java
@@ -19,10 +19,8 @@
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -30,7 +28,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-import org.apache.hadoop.yarn.server.federation.policies.exceptions.UnknownSubclusterException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -40,8 +37,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
*/
public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
- private Set<SubClusterId> knownClusterIds = new HashSet<>();
-
@Override
public void reinitialize(
FederationPolicyInitializationContext policyContext)
@@ -65,7 +60,6 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
// simply broadcast the resource request to all sub-clusters
for (SubClusterId subClusterId : activeSubclusters.keySet()) {
answer.put(subClusterId, resourceRequests);
- knownClusterIds.add(subClusterId);
}
return answer;
@@ -74,11 +68,6 @@ public class BroadcastAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
@Override
public void notifyOfResponse(SubClusterId subClusterId,
AllocateResponse response) throws YarnException {
- if (!knownClusterIds.contains(subClusterId)) {
- throw new UnknownSubclusterException(
- "The response is received from a subcluster that is unknown to this "
- + "policy.");
- }
// stateless policy does not care about responses
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
index 3783df6..4500606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -18,10 +18,8 @@
package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -38,8 +36,6 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
*/
public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
- private Set<SubClusterId> knownClusterIds = new HashSet<>();
-
@Override
public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdb08442/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
index a21f53d..df5da85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestBroadcastAMRMProxyFederationPolicy.java
@@ -89,7 +89,7 @@ public class TestBroadcastAMRMProxyFederationPolicy
}
@Test
- public void testNotifyOfResponse() throws Exception {
+ public void testNotifyOfResponseFromUnknownSubCluster() throws Exception {
String[] hosts = new String[] {"host1", "host2" };
List<ResourceRequest> resourceRequests = FederationPoliciesTestUtil
.createResourceRequests(hosts, 2 * 1024, 2, 1, 3, null, false);
@@ -97,13 +97,8 @@ public class TestBroadcastAMRMProxyFederationPolicy
((FederationAMRMProxyPolicy) getPolicy())
.splitResourceRequests(resourceRequests);
- try {
- ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
- SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
- Assert.fail();
- } catch (FederationPolicyException f) {
- System.out.println("Expected: " + f.getMessage());
- }
+ ((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
+ SubClusterId.newInstance("sc3"), mock(AllocateResponse.class));
((FederationAMRMProxyPolicy) getPolicy()).notifyOfResponse(
SubClusterId.newInstance("sc1"), mock(AllocateResponse.class));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-192:Create new SCMCommand to
request a replication of a container. Contributed by Elek Marton
Posted by gi...@apache.org.
HDDS-192:Create new SCMCommand to request a replication of a container. Contributed by Elek Marton
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/238fe00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/238fe00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/238fe00a
Branch: refs/heads/HADOOP-15461
Commit: 238fe00ad2692154f6a382f35735169ee5e4af2c
Parents: 35ec940
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Jun 25 21:12:05 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Jun 25 21:12:05 2018 -0700
----------------------------------------------------------------------
.../statemachine/DatanodeStateMachine.java | 3 +
.../ReplicateContainerCommandHandler.java | 67 ++++++++++++++
.../states/endpoint/HeartbeatEndpointTask.java | 12 +++
.../commands/ReplicateContainerCommand.java | 94 ++++++++++++++++++++
.../StorageContainerDatanodeProtocol.proto | 12 ++-
.../scm/server/SCMDatanodeProtocolServer.java | 11 +++
.../TestReplicateContainerHandler.java | 71 +++++++++++++++
7 files changed, 269 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index dc4e673..b073d7b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.CommandDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
+import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
+ .ReplicateContainerCommandHandler;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
@@ -95,6 +97,7 @@ public class DatanodeStateMachine implements Closeable {
.addHandler(new CloseContainerCommandHandler())
.addHandler(new DeleteBlocksCommandHandler(
container.getContainerManager(), conf))
+ .addHandler(new ReplicateContainerCommandHandler())
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
new file mode 100644
index 0000000..b4e83b7
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine
+ .SCMConnectionManager;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Command handler to copy containers from sources.
+ */
+public class ReplicateContainerCommandHandler implements CommandHandler {
+ static final Logger LOG =
+ LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
+
+ private int invocationCount;
+
+ private long totalTime;
+
+ @Override
+ public void handle(SCMCommand command, OzoneContainer container,
+ StateContext context, SCMConnectionManager connectionManager) {
+ LOG.warn("Replicate command is not yet handled");
+
+ }
+
+ @Override
+ public SCMCommandProto.Type getCommandType() {
+ return Type.replicateContainerCommand;
+ }
+
+ @Override
+ public int getInvocationCount() {
+ return this.invocationCount;
+ }
+
+ @Override
+ public long getAverageRunTime() {
+ if (invocationCount > 0) {
+ return totalTime / invocationCount;
+ }
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
index 1ee6375..260a245 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.ozone.container.common.statemachine
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -196,6 +198,16 @@ public class HeartbeatEndpointTask
}
this.context.addCommand(closeContainer);
break;
+ case replicateContainerCommand:
+ ReplicateContainerCommand replicateContainerCommand =
+ ReplicateContainerCommand.getFromProtobuf(
+ commandResponseProto.getReplicateContainerCommandProto());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received SCM container replicate request for container {}",
+ replicateContainerCommand.getContainerID());
+ }
+ this.context.addCommand(replicateContainerCommand);
+ break;
default:
throw new IllegalArgumentException("Unknown response : "
+ commandResponseProto.getCommandType().name());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
new file mode 100644
index 0000000..834318b
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto
+ .Builder;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * SCM command to request replication of a container.
+ */
+public class ReplicateContainerCommand
+ extends SCMCommand<ReplicateContainerCommandProto> {
+
+ private final long containerID;
+
+ private final List<DatanodeDetails> sourceDatanodes;
+
+ public ReplicateContainerCommand(long containerID,
+ List<DatanodeDetails> sourceDatanodes) {
+ this.containerID = containerID;
+ this.sourceDatanodes = sourceDatanodes;
+ }
+
+ @Override
+ public Type getType() {
+ return SCMCommandProto.Type.replicateContainerCommand;
+ }
+
+ @Override
+ public byte[] getProtoBufMessage() {
+ return getProto().toByteArray();
+ }
+
+ public ReplicateContainerCommandProto getProto() {
+ Builder builder = ReplicateContainerCommandProto.newBuilder()
+ .setContainerID(containerID);
+ for (DatanodeDetails dd : sourceDatanodes) {
+ builder.addSources(dd.getProtoBufMessage());
+ }
+ return builder.build();
+ }
+
+ public static ReplicateContainerCommand getFromProtobuf(
+ ReplicateContainerCommandProto protoMessage) {
+ Preconditions.checkNotNull(protoMessage);
+
+ List<DatanodeDetails> datanodeDetails =
+ protoMessage.getSourcesList()
+ .stream()
+ .map(DatanodeDetails::getFromProtoBuf)
+ .collect(Collectors.toList());
+
+ return new ReplicateContainerCommand(protoMessage.getContainerID(),
+ datanodeDetails);
+
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public List<DatanodeDetails> getSourceDatanodes() {
+ return sourceDatanodes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index f6aba05..54230c1 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -172,6 +172,7 @@ message SCMCommandProto {
deleteBlocksCommand = 2;
closeContainerCommand = 3;
deleteContainerCommand = 4;
+ replicateContainerCommand = 5;
}
// TODO: once we start using protoc 3.x, refactor this message using "oneof"
required Type commandType = 1;
@@ -179,6 +180,7 @@ message SCMCommandProto {
optional DeleteBlocksCommandProto deleteBlocksCommandProto = 3;
optional CloseContainerCommandProto closeContainerCommandProto = 4;
optional DeleteContainerCommandProto deleteContainerCommandProto = 5;
+ optional ReplicateContainerCommandProto replicateContainerCommandProto = 6;
}
/**
@@ -227,13 +229,21 @@ message CloseContainerCommandProto {
}
/**
-This command asks the datanode to close a specific container.
+This command asks the datanode to delete a specific container.
*/
message DeleteContainerCommandProto {
required int64 containerID = 1;
}
/**
+This command asks the datanode to replicate a container from specific sources.
+*/
+message ReplicateContainerCommandProto {
+ required int64 containerID = 1;
+ repeated DatanodeDetailsProto sources = 2;
+}
+
+/**
* Protocol used from a datanode to StorageContainerManager.
*
* Please see the request and response messages for details of the RPC calls.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 7d16161..eb5ce1a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
@@ -63,6 +64,9 @@ import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto
.Type.deleteBlocksCommand;
import static org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type
+ .replicateContainerCommand;
+import static org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto
.Type.reregisterCommand;
@@ -77,6 +81,7 @@ import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
import org.apache.hadoop.ozone.protocolPB
@@ -293,6 +298,12 @@ public class SCMDatanodeProtocolServer implements
.setCloseContainerCommandProto(
((CloseContainerCommand) cmd).getProto())
.build();
+ case replicateContainerCommand:
+ return builder
+ .setCommandType(replicateContainerCommand)
+ .setReplicateContainerCommandProto(
+ ((ReplicateContainerCommand)cmd).getProto())
+ .build();
default:
throw new IllegalArgumentException("Not implemented");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/238fe00a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
new file mode 100644
index 0000000..a5b101f
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerHandler.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_SIZE_GB;
+import org.junit.Test;
+
+/**
+ * Tests the behavior of the datanode, when replicate container command is
+ * received.
+ */
+public class TestReplicateContainerHandler {
+
+ @Test
+ public void test() throws IOException, TimeoutException, InterruptedException,
+ OzoneException {
+
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(ReplicateContainerCommandHandler.LOG);
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_SCM_CONTAINER_SIZE_GB, "1");
+ MiniOzoneCluster cluster =
+ MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+ cluster.waitForClusterToBeReady();
+
+ DatanodeDetails datanodeDetails =
+ cluster.getHddsDatanodes().get(0).getDatanodeDetails();
+ //send the order to replicate the container
+ cluster.getStorageContainerManager().getScmNodeManager()
+ .addDatanodeCommand(datanodeDetails.getUuid(),
+ new ReplicateContainerCommand(1L,
+ new ArrayList<>()));
+
+ //TODO: here we test only the serialization/unserialization as
+ // the implementation is not yet done
+ GenericTestUtils
+ .waitFor(() -> logCapturer.getOutput().contains("not yet handled"), 500,
+ 5 * 1000);
+
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-8443. Total #VCores in
cluster metrics is wrong when CapacityScheduler reserved some containers.
Contributed by Tao Yang.
Posted by gi...@apache.org.
YARN-8443. Total #VCores in cluster metrics is wrong when CapacityScheduler reserved some containers. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/440140ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/440140ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/440140ce
Branch: refs/heads/HADOOP-15461
Commit: 440140cea6718229094a3d2b97b9b9bd28b95d9b
Parents: e16e5b3
Author: Weiwei Yang <ww...@apache.org>
Authored: Mon Jun 25 09:15:31 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Mon Jun 25 09:15:31 2018 +0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/440140ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 84f70d9..69d88aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -101,7 +101,7 @@ public class ClusterMetricsInfo {
CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores =
- availableVirtualCores + allocatedVirtualCores + containersReserved;
+ availableVirtualCores + allocatedVirtualCores + reservedVirtualCores;
// TODO, add support of other schedulers to get total used resources
// across partition.
if (cs.getRootQueue() != null
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-8214. Change default
RegistryDNS port. Contributed by Billie Rinaldi
Posted by gi...@apache.org.
YARN-8214. Change default RegistryDNS port.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e586330
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e586330
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e586330
Branch: refs/heads/HADOOP-15461
Commit: 3e586330eb2c7db0b884fe328b171fb27ce545fa
Parents: 238fe00
Author: Eric Yang <ey...@apache.org>
Authored: Tue Jun 26 14:25:15 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Jun 26 14:25:15 2018 -0400
----------------------------------------------------------------------
.../hadoop/registry/client/api/RegistryConstants.java | 2 +-
.../src/site/markdown/yarn-service/RegistryDNS.md | 10 +++++-----
.../src/site/markdown/yarn-service/ServiceDiscovery.md | 6 +++---
3 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index cfa2d65..bd97a5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -95,7 +95,7 @@ public interface RegistryConstants {
/**
* Default DNS port number.
*/
- int DEFAULT_DNS_PORT = 5353;
+ int DEFAULT_DNS_PORT = 5335;
/**
* DNSSEC Enabled?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
index 2307e5c..642d26e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -58,7 +58,7 @@ primary DNS server can be configured to forward a zone to the registry DNS
server.
2. The DNS Server exposes a port that can receive both TCP and UDP requests per
DNS standards. The default port for DNS protocols is not in the restricted
-range (5353). However, existing DNS assets may only allow zone forwarding to
+range (5335). However, existing DNS assets may only allow zone forwarding to
non-custom ports. To support this, the registry DNS server can be started in
privileged mode.
@@ -136,7 +136,7 @@ standard DNS requests from users or other DNS servers (for example, DNS servers
RegistryDNS service configured as a forwarder).
## Start the DNS Server
-By default, the DNS server runs on non-privileged port `5353`. Start the server
+By default, the DNS server runs on non-privileged port `5335`. Start the server
with:
```
yarn --daemon start registrydns
@@ -157,7 +157,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
-| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5353. |
+| hadoop.registry.dns.bind-port | The port number for the DNS listener. The default port is 5335. |
| hadoop.registry.dns.dnssec.enabled | Indicates whether the DNSSEC support is enabled. Default is false. |
| hadoop.registry.dns.public-key | The base64 representation of the server’s public key. Leveraged for creating the DNSKEY Record provided for DNSSEC client requests. |
| hadoop.registry.dns.private-key-file | The path to the standard DNSSEC private key file. Must only be readable by the DNS launching identity. See [dnssec-keygen](https://ftp.isc.org/isc/bind/cur/9.9/doc/arm/man.dnssec-keygen.html) documentation. |
@@ -174,10 +174,10 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
</property>
<property>
- <description>The port number for the DNS listener. The default port is 5353.
+ <description>The port number for the DNS listener. The default port is 5335.
If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
<name>hadoop.registry.dns.bind-port</name>
- <value>5353</value>
+ <value>5335</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e586330/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index 7ee16dd..6b93f3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -94,10 +94,10 @@ section of [Registry DNS](RegistryDNS.html).
</property>
<property>
- <description>The port number for the DNS listener. The default port is 5353.
+ <description>The port number for the DNS listener. The default port is 5335.
If the standard privileged port 53 is used, make sure start the DNS with jsvc support.</description>
<name>hadoop.registry.dns.bind-port</name>
- <value>5353</value>
+ <value>5335</value>
</property>
<property>
@@ -135,7 +135,7 @@ To configure Registry DNS to serve reverse lookup for `172.17.0.0/24`
</property>
```
## Start Registry DNS Server
-By default, the DNS server runs on non-privileged port `5353`. Start the server
+By default, the DNS server runs on non-privileged port `5335`. Start the server
with:
```
yarn --daemon start registrydns
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: MAPREDUCE-7114. Make
FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.
Posted by gi...@apache.org.
MAPREDUCE-7114. Make FrameworkUploader symlink ignore improvement. Contributed by Gergo Repas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae055622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae055622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae055622
Branch: refs/heads/HADOOP-15461
Commit: ae055622edeb3cbf82baa6ed952fc2abc84c021e
Parents: 55fad6a
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Fri Jun 22 13:05:41 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Fri Jun 22 13:05:41 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapred/uploader/FrameworkUploader.java | 2 +-
.../hadoop/mapred/uploader/TestFrameworkUploader.java | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index 5316f38..d2116c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -409,7 +409,7 @@ public class FrameworkUploader implements Runnable {
linkPath == null ? null : linkPath.getParent();
java.nio.file.Path normalizedLinkPath =
linkPathParent == null ? null : linkPathParent.normalize();
- if (normalizedLinkPath != null && jarParent.equals(
+ if (normalizedLinkPath != null && jarParent.normalize().equals(
normalizedLinkPath)) {
LOG.info(String.format("Ignoring same directory link %s to %s",
jarPath.toString(), link.toString()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae055622/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
index c12902c..9c72f72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
@@ -440,6 +440,19 @@ public class TestFrameworkUploader {
}
Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
+ // Create a symlink to the target with /./ in the path
+ symlinkToTarget = new File(parent.getAbsolutePath() +
+ "/./symlinkToTarget2.txt");
+ try {
+ Files.createSymbolicLink(
+ Paths.get(symlinkToTarget.getAbsolutePath()),
+ Paths.get(targetFile.getAbsolutePath()));
+ } catch (UnsupportedOperationException e) {
+ // Symlinks are not supported, so ignore the test
+ Assume.assumeTrue(false);
+ }
+ Assert.assertTrue(uploader.checkSymlink(symlinkToTarget));
+
// Create a symlink outside the current directory
File symlinkOutside = new File(parent, "symlinkToParent.txt");
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HADOOP-15416. Clear error message
in S3Guard diff if source not found. Contributed by Gabor Bota.
Posted by gi...@apache.org.
HADOOP-15416. Clear error message in S3Guard diff if source not found. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55fad6a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55fad6a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55fad6a3
Branch: refs/heads/HADOOP-15461
Commit: 55fad6a3de3125d9e7e2e9a5f8fa5b1b22a1de60
Parents: 6432128
Author: Sean Mackrory <ma...@apache.org>
Authored: Fri Jun 22 11:36:36 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Fri Jun 22 11:36:56 2018 -0600
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55fad6a3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index fbffba9..ac10e08 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -805,7 +805,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
private void compareDir(FileStatus msDir, FileStatus s3Dir,
PrintStream out) throws IOException {
- Preconditions.checkArgument(!(msDir == null && s3Dir == null));
+ Preconditions.checkArgument(!(msDir == null && s3Dir == null),
+ "The path does not exist in metadata store and on s3.");
+
if (msDir != null && s3Dir != null) {
Preconditions.checkArgument(msDir.getPath().equals(s3Dir.getPath()),
String.format("The path from metadata store and s3 are different:" +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDDS-170. Fix
TestBlockDeletingService#testBlockDeletionTimeout. Contributed by Lokesh
Jain.
Posted by gi...@apache.org.
HDDS-170. Fix TestBlockDeletingService#testBlockDeletionTimeout. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e305476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e305476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e305476
Branch: refs/heads/HADOOP-15461
Commit: 1e30547642c7c6c014745862dd06f90f091f90b6
Parents: e9ec3d7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:56:45 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:56:45 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/utils/BackgroundService.java | 2 +-
.../statemachine/background/BlockDeletingService.java | 8 ++++----
.../hadoop/ozone/container/ozoneimpl/OzoneContainer.java | 2 +-
.../testutils/BlockDeletingServiceTestImpl.java | 4 ++--
.../ozone/container/common/TestBlockDeletingService.java | 11 +++++++----
5 files changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
index 431da64..5718008 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/BackgroundService.java
@@ -126,7 +126,7 @@ public abstract class BackgroundService {
try {
// Collect task results
BackgroundTaskResult result = serviceTimeout > 0
- ? taskResultFuture.get(serviceTimeout, TimeUnit.MILLISECONDS)
+ ? taskResultFuture.get(serviceTimeout, unit)
: taskResultFuture.get();
if (LOG.isDebugEnabled()) {
LOG.debug("task execution result size {}", result.getSize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
index 63f57b4..bff5913 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
@@ -83,10 +83,10 @@ public class BlockDeletingService extends BackgroundService{
private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
public BlockDeletingService(ContainerManager containerManager,
- long serviceInterval, long serviceTimeout, Configuration conf) {
- super("BlockDeletingService", serviceInterval,
- TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE,
- serviceTimeout);
+ long serviceInterval, long serviceTimeout, TimeUnit unit,
+ Configuration conf) {
+ super("BlockDeletingService", serviceInterval, unit,
+ BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
this.containerManager = containerManager;
this.conf = conf;
this.blockLimitPerTask = conf.getInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 4156f5a..7931f6f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -122,7 +122,7 @@ public class OzoneContainer {
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
this.blockDeletingService = new BlockDeletingService(manager,
- svcInterval, serviceTimeout, ozoneConfig);
+ svcInterval, serviceTimeout, TimeUnit.MILLISECONDS, ozoneConfig);
this.dispatcher = new Dispatcher(manager, this.ozoneConfig);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
index 86888aa..7c12945 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/testutils/BlockDeletingServiceTestImpl.java
@@ -44,8 +44,8 @@ public class BlockDeletingServiceTestImpl
public BlockDeletingServiceTestImpl(ContainerManager containerManager,
int serviceInterval, Configuration conf) {
- super(containerManager, serviceInterval,
- SERVICE_TIMEOUT_IN_MILLISECONDS, conf);
+ super(containerManager, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
+ TimeUnit.MILLISECONDS, conf);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e305476/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index 0686e4e..8d01c80 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -280,10 +280,11 @@ public class TestBlockDeletingService {
ContainerManager containerManager = createContainerManager(conf);
createToDeleteBlocks(containerManager, conf, 1, 3, 1, chunksDir);
- // set timeout value as 1ms to trigger timeout behavior
+ // set timeout value as 1ns to trigger timeout behavior
long timeout = 1;
- BlockDeletingService svc =
- new BlockDeletingService(containerManager, 1000, timeout, conf);
+ BlockDeletingService svc = new BlockDeletingService(containerManager,
+ TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
+ conf);
svc.start();
LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
@@ -303,7 +304,9 @@ public class TestBlockDeletingService {
// test for normal case that doesn't have timeout limitation
timeout = 0;
createToDeleteBlocks(containerManager, conf, 1, 3, 1, chunksDir);
- svc = new BlockDeletingService(containerManager, 1000, timeout, conf);
+ svc = new BlockDeletingService(containerManager,
+ TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
+ conf);
svc.start();
// get container meta data
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-195. Create generic
CommandWatcher utility. Contributed by Elek, Marton.
Posted by gi...@apache.org.
HDDS-195. Create generic CommandWatcher utility.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85627e2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85627e2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85627e2c
Branch: refs/heads/HADOOP-15461
Commit: 85627e2cba91a61d675d20cdb35e188c6c81e3f2
Parents: ddbff7c
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Jun 27 23:31:32 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 23:39:33 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/server/events/EventWatcher.java | 157 +++++++++++++
.../server/events/IdentifiableEventPayload.java | 30 +++
.../hdds/server/events/EventHandlerStub.java | 38 ++++
.../hdds/server/events/TestEventWatcher.java | 220 +++++++++++++++++++
4 files changed, 445 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
new file mode 100644
index 0000000..19fddde
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/EventWatcher.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.ozone.lease.Lease;
+import org.apache.hadoop.ozone.lease.LeaseAlreadyExistException;
+import org.apache.hadoop.ozone.lease.LeaseExpiredException;
+import org.apache.hadoop.ozone.lease.LeaseManager;
+import org.apache.hadoop.ozone.lease.LeaseNotFoundException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Event watcher the (re)send a message after timeout.
+ * <p>
+ * Event watcher will send the tracked payload/event after a timeout period
+ * unless a confirmation from the original event (completion event) is arrived.
+ *
+ * @param <TIMEOUT_PAYLOAD> The type of the events which are tracked.
+ * @param <COMPLETION_PAYLOAD> The type of event which could cancel the
+ * tracking.
+ */
+@SuppressWarnings("CheckStyle")
+public abstract class EventWatcher<TIMEOUT_PAYLOAD extends
+ IdentifiableEventPayload,
+ COMPLETION_PAYLOAD extends IdentifiableEventPayload> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EventWatcher.class);
+
+ private final Event<TIMEOUT_PAYLOAD> startEvent;
+
+ private final Event<COMPLETION_PAYLOAD> completionEvent;
+
+ private final LeaseManager<UUID> leaseManager;
+
+ protected final Map<UUID, TIMEOUT_PAYLOAD> trackedEventsByUUID =
+ new ConcurrentHashMap<>();
+
+ protected final Set<TIMEOUT_PAYLOAD> trackedEvents = new HashSet<>();
+
+ public EventWatcher(Event<TIMEOUT_PAYLOAD> startEvent,
+ Event<COMPLETION_PAYLOAD> completionEvent,
+ LeaseManager<UUID> leaseManager) {
+ this.startEvent = startEvent;
+ this.completionEvent = completionEvent;
+ this.leaseManager = leaseManager;
+
+ }
+
+ public void start(EventQueue queue) {
+
+ queue.addHandler(startEvent, this::handleStartEvent);
+
+ queue.addHandler(completionEvent, (completionPayload, publisher) -> {
+ UUID uuid = completionPayload.getUUID();
+ try {
+ handleCompletion(uuid, publisher);
+ } catch (LeaseNotFoundException e) {
+ //It's already done. Too late, we already retried it.
+ //Not a real problem.
+ LOG.warn("Completion event without active lease. UUID={}", uuid);
+ }
+ });
+
+ }
+
+ private synchronized void handleStartEvent(TIMEOUT_PAYLOAD payload,
+ EventPublisher publisher) {
+ UUID identifier = payload.getUUID();
+ trackedEventsByUUID.put(identifier, payload);
+ trackedEvents.add(payload);
+ try {
+ Lease<UUID> lease = leaseManager.acquire(identifier);
+ try {
+ lease.registerCallBack(() -> {
+ handleTimeout(publisher, identifier);
+ return null;
+ });
+
+ } catch (LeaseExpiredException e) {
+ handleTimeout(publisher, identifier);
+ }
+ } catch (LeaseAlreadyExistException e) {
+ //No problem at all. But timer is not reset.
+ }
+ }
+
+ private synchronized void handleCompletion(UUID uuid,
+ EventPublisher publisher) throws LeaseNotFoundException {
+ leaseManager.release(uuid);
+ TIMEOUT_PAYLOAD payload = trackedEventsByUUID.remove(uuid);
+ trackedEvents.remove(payload);
+ onFinished(publisher, payload);
+ }
+
+ private synchronized void handleTimeout(EventPublisher publisher,
+ UUID identifier) {
+ TIMEOUT_PAYLOAD payload = trackedEventsByUUID.remove(identifier);
+ trackedEvents.remove(payload);
+ onTimeout(publisher, payload);
+ }
+
+
+ /**
+ * Check if a specific payload is in-progress.
+ */
+ public synchronized boolean contains(TIMEOUT_PAYLOAD payload) {
+ return trackedEvents.contains(payload);
+ }
+
+ public synchronized boolean remove(TIMEOUT_PAYLOAD payload) {
+ try {
+ leaseManager.release(payload.getUUID());
+ } catch (LeaseNotFoundException e) {
+ LOG.warn("Completion event without active lease. UUID={}",
+ payload.getUUID());
+ }
+ trackedEventsByUUID.remove(payload.getUUID());
+ return trackedEvents.remove(payload);
+
+ }
+
+ abstract void onTimeout(EventPublisher publisher, TIMEOUT_PAYLOAD payload);
+
+ abstract void onFinished(EventPublisher publisher, TIMEOUT_PAYLOAD payload);
+
+ public List<TIMEOUT_PAYLOAD> getTimeoutEvents(
+ Predicate<? super TIMEOUT_PAYLOAD> predicate) {
+ return trackedEventsByUUID.values().stream().filter(predicate)
+ .collect(Collectors.toList());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
new file mode 100644
index 0000000..e73e30f
--- /dev/null
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/events/IdentifiableEventPayload.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.UUID;
+
+/**
+ * Event with an additional unique identifier.
+ *
+ */
+public interface IdentifiableEventPayload {
+
+ UUID getUUID();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
new file mode 100644
index 0000000..3f34a70
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/EventHandlerStub.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Dummy class for testing to collect all the received events.
+ */
+public class EventHandlerStub<PAYLOAD> implements EventHandler<PAYLOAD> {
+
+ private List<PAYLOAD> receivedEvents = new ArrayList<>();
+
+ @Override
+ public void onMessage(PAYLOAD payload, EventPublisher publisher) {
+ receivedEvents.add(payload);
+ }
+
+ public List<PAYLOAD> getReceivedEvents() {
+ return receivedEvents;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85627e2c/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
new file mode 100644
index 0000000..1731350
--- /dev/null
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventWatcher.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.server.events;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+
+import org.apache.hadoop.ozone.lease.LeaseManager;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test the basic functionality of event watcher.
+ */
+public class TestEventWatcher {
+
+ private static final TypedEvent<UnderreplicatedEvent> WATCH_UNDER_REPLICATED =
+ new TypedEvent<>(UnderreplicatedEvent.class);
+
+ private static final TypedEvent<UnderreplicatedEvent> UNDER_REPLICATED =
+ new TypedEvent<>(UnderreplicatedEvent.class);
+
+ private static final TypedEvent<ReplicationCompletedEvent>
+ REPLICATION_COMPLETED = new TypedEvent<>(ReplicationCompletedEvent.class);
+
+ LeaseManager<UUID> leaseManager;
+
+ @Before
+ public void startLeaseManager() {
+ leaseManager = new LeaseManager<>(2000l);
+ leaseManager.start();
+ }
+
+ @After
+ public void stopLeaseManager() {
+ leaseManager.shutdown();
+ }
+
+
+ @Test
+ public void testEventHandling() throws InterruptedException {
+
+ EventQueue queue = new EventQueue();
+
+ EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+ replicationWatcher = createEventWatcher();
+
+ EventHandlerStub<UnderreplicatedEvent> underReplicatedEvents =
+ new EventHandlerStub<>();
+
+ queue.addHandler(UNDER_REPLICATED, underReplicatedEvents);
+
+ replicationWatcher.start(queue);
+
+ UUID uuid1 = UUID.randomUUID();
+ UUID uuid2 = UUID.randomUUID();
+
+ queue.fireEvent(WATCH_UNDER_REPLICATED,
+ new UnderreplicatedEvent(uuid1, "C1"));
+
+ queue.fireEvent(WATCH_UNDER_REPLICATED,
+ new UnderreplicatedEvent(uuid2, "C2"));
+
+ Assert.assertEquals(0, underReplicatedEvents.getReceivedEvents().size());
+
+ Thread.sleep(1000);
+
+ queue.fireEvent(REPLICATION_COMPLETED,
+ new ReplicationCompletedEvent(uuid1, "C2", "D1"));
+
+ Assert.assertEquals(0, underReplicatedEvents.getReceivedEvents().size());
+
+ Thread.sleep(1500);
+
+ queue.processAll(1000L);
+
+ Assert.assertEquals(1, underReplicatedEvents.getReceivedEvents().size());
+ Assert.assertEquals(uuid2,
+ underReplicatedEvents.getReceivedEvents().get(0).UUID);
+
+ }
+
+ @Test
+ public void testInprogressFilter() throws InterruptedException {
+
+ EventQueue queue = new EventQueue();
+
+ EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+ replicationWatcher = createEventWatcher();
+
+ EventHandlerStub<UnderreplicatedEvent> underReplicatedEvents =
+ new EventHandlerStub<>();
+
+ queue.addHandler(UNDER_REPLICATED, underReplicatedEvents);
+
+ replicationWatcher.start(queue);
+
+ UnderreplicatedEvent event1 =
+ new UnderreplicatedEvent(UUID.randomUUID(), "C1");
+
+ queue.fireEvent(WATCH_UNDER_REPLICATED, event1);
+
+ queue.fireEvent(WATCH_UNDER_REPLICATED,
+ new UnderreplicatedEvent(UUID.randomUUID(), "C2"));
+
+ queue.fireEvent(WATCH_UNDER_REPLICATED,
+ new UnderreplicatedEvent(UUID.randomUUID(), "C1"));
+
+ queue.processAll(1000L);
+ Thread.sleep(1000L);
+ List<UnderreplicatedEvent> c1todo = replicationWatcher
+ .getTimeoutEvents(e -> e.containerId.equalsIgnoreCase("C1"));
+
+ Assert.assertEquals(2, c1todo.size());
+ Assert.assertTrue(replicationWatcher.contains(event1));
+ Thread.sleep(1500L);
+
+ c1todo = replicationWatcher
+ .getTimeoutEvents(e -> e.containerId.equalsIgnoreCase("C1"));
+ Assert.assertEquals(0, c1todo.size());
+ Assert.assertFalse(replicationWatcher.contains(event1));
+
+
+ }
+
+ private EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>
+ createEventWatcher() {
+ return new EventWatcher<UnderreplicatedEvent, ReplicationCompletedEvent>(
+ WATCH_UNDER_REPLICATED, REPLICATION_COMPLETED, leaseManager) {
+
+ @Override
+ void onTimeout(EventPublisher publisher, UnderreplicatedEvent payload) {
+ publisher.fireEvent(UNDER_REPLICATED, payload);
+ }
+
+ @Override
+ void onFinished(EventPublisher publisher, UnderreplicatedEvent payload) {
+ //Good job. We did it.
+ }
+ };
+ }
+
+ private static class ReplicationCompletedEvent
+ implements IdentifiableEventPayload {
+
+ private final UUID UUID;
+
+ private final String containerId;
+
+ private final String datanodeId;
+
+ public ReplicationCompletedEvent(UUID UUID, String containerId,
+ String datanodeId) {
+ this.UUID = UUID;
+ this.containerId = containerId;
+ this.datanodeId = datanodeId;
+ }
+
+ public UUID getUUID() {
+ return UUID;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationCompletedEvent that = (ReplicationCompletedEvent) o;
+ return Objects.equals(containerId, that.containerId) && Objects
+ .equals(datanodeId, that.datanodeId);
+ }
+
+ @Override
+ public int hashCode() {
+
+ return Objects.hash(containerId, datanodeId);
+ }
+ }
+
+ private static class UnderreplicatedEvent
+
+ implements IdentifiableEventPayload {
+
+ private final UUID UUID;
+
+ private final String containerId;
+
+ public UnderreplicatedEvent(UUID UUID, String containerId) {
+ this.containerId = containerId;
+ this.UUID = UUID;
+ }
+
+ public UUID getUUID() {
+ return UUID;
+ }
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDFS-13705:The native ISA-L
library loading failure should be made warning rather than an error message.
Contributed by Shashikant Banerjee.
Posted by gi...@apache.org.
HDFS-13705:The native ISA-L library loading failure should be made warning rather than an error message. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3fa83a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3fa83a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3fa83a4
Branch: refs/heads/HADOOP-15461
Commit: d3fa83a44b01c85f39bfb4deaf2972912ac61ca3
Parents: 85627e2
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Jun 28 09:21:56 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Jun 28 09:21:56 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fa83a4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
index 3d6867a..ec317ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeNative.java
@@ -46,7 +46,7 @@ public final class ErasureCodeNative {
loadLibrary();
} catch (Throwable t) {
problem = "Loading ISA-L failed: " + t.getMessage();
- LOG.error("Loading ISA-L failed", t);
+ LOG.warn(problem);
}
LOADING_FAILURE_REASON = problem;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HADOOP-15495. Upgrade commons-lang
version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by
Takanobu Asanuma.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 0ab8696..116827d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -60,7 +60,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 49981ed..f0ffb44 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -23,7 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index ac10e08..527697f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -39,7 +39,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index bfac975..9e2f34d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3native;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -31,7 +31,7 @@ import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.Objects;
-import static org.apache.commons.lang.StringUtils.equalsIgnoreCase;
+import static org.apache.commons.lang3.StringUtils.equalsIgnoreCase;
/**
* Class to aid logging in to S3 endpoints.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index f1799ac..aa6b5d8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -22,8 +22,8 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.S3ClientOptions;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.reflect.FieldUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
index 50c9fb5..a8a78f6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
index 4c953bd..c170830 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKeyBlockOutputStream.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.s3a;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index d259bf1..869997b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.s3a;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
index b8610d6..90e8894 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
@@ -28,7 +28,7 @@ import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 3e149a6..79e8a69 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -42,7 +42,7 @@ import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
index 12c2e3f..a68e6ac 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java
@@ -25,7 +25,7 @@ import java.net.URISyntaxException;
import com.microsoft.azure.datalake.store.oauth2.DeviceCodeTokenProvider;
import com.microsoft.azure.datalake.store.oauth2.MsiTokenProvider;
-import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 9396a51..197ab22 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -41,7 +41,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
index 9a85308..5f051ef 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java
@@ -44,7 +44,7 @@ import java.util.concurrent.atomic.AtomicReference;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.commons.codec.binary.Base64;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
index aa7ac2e..850e552 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/ClientThrottlingAnalyzer.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.fs.azure;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index dfc881a..5202762 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -48,7 +48,7 @@ import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
index 68ddcdf..6e98755 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
@@ -36,7 +36,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
index ea77510..76ced3b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java
@@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azure.security.Constants;
import org.apache.hadoop.io.retry.RetryPolicy;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
index a0204be..f4ec172 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureWasbRemoteCallHelper.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.azure;
-import org.apache.commons.lang.Validate;
+import org.apache.commons.lang3.Validate;
import org.apache.hadoop.fs.azure.security.Constants;
import org.apache.hadoop.fs.azure.security.SpnegoToken;
import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
index 0aa9393..f54a2e1 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestBlobDataValidation.java
@@ -29,7 +29,7 @@ import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.util.Arrays;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
index 29611bf..4e88b45 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestNativeAzureFileSystemAppend.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
index d5f6437..1739cff 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
@@ -36,7 +36,7 @@ import java.util.TimeZone;
import java.util.List;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.net.URLCodec;
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
import org.apache.http.client.utils.URIBuilder;
import com.microsoft.azure.storage.AccessCondition;
@@ -339,7 +339,7 @@ public class MockStorageInterface extends StorageInterface {
@Override
public StorageUri getStorageUri() {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
}
@@ -590,20 +590,20 @@ public class MockStorageInterface extends StorageInterface {
@Override
public void create(long length, BlobRequestOptions options,
OperationContext opContext) throws StorageException {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
@Override
public void uploadPages(InputStream sourceStream, long offset, long length,
BlobRequestOptions options, OperationContext opContext)
throws StorageException, IOException {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
@Override
public ArrayList<PageRange> downloadPageRanges(BlobRequestOptions options,
OperationContext opContext) throws StorageException {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
@Override
@@ -622,7 +622,7 @@ public class MockStorageInterface extends StorageInterface {
@Override
public StorageUri getStorageUri() {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
index 820ce4f..b8cf5ba 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockCompaction.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.azure;
import com.microsoft.azure.storage.blob.BlockEntry;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
index 2fbbcd1..8d2a104 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java
@@ -28,7 +28,7 @@ import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index ea99016..9db0eb5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.tools;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
index 668b594..e49feb5 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
@@ -28,7 +28,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
index faa4aa2..c486bdb 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
@@ -22,7 +22,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.EnumSet;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
index 9568171..cf6da25 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ClusterSummarizer.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.mapred.gridmix;
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
index 8f9d434..973838a 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ExecutionSummarizer.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.mapred.gridmix;
import java.io.IOException;
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -76,7 +76,7 @@ class ExecutionSummarizer implements StatListener<JobStats> {
startTime = System.currentTimeMillis();
// flatten the args string and store it
commandLineArgs =
- org.apache.commons.lang.StringUtils.join(args, ' ');
+ org.apache.commons.lang3.StringUtils.join(args, ' ');
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
index 4271742..73662bf 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobFactory.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.mapred.gridmix;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
index 877d434..494b9a1 100644
--- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
+++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/RandomTextDataGenerator.java
@@ -21,7 +21,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
index afafd55..2accbac 100644
--- a/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
+++ b/hadoop-tools/hadoop-resourceestimator/src/main/java/org/apache/hadoop/resourceestimator/translator/impl/BaseLogParser.java
@@ -30,7 +30,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.CharSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.resourceestimator.common.api.RecurrenceId;
import org.apache.hadoop.resourceestimator.common.api.ResourceSkyline;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
index 5856626..e6d09dc 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/anonymization/WordListAnonymizerUtility.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.tools.rumen.anonymization;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
/**
* Utility class to handle commonly performed tasks in a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
index 1c92caf..02fd48a 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
@@ -25,7 +25,7 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRJobConfig;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
index bd50ae0..e1f6da5 100644
--- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
+++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUnconsumedInput.java
@@ -25,7 +25,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-186. Create under replicated
queue. Contributed by Ajay Kumar.
Posted by gi...@apache.org.
HDDS-186. Create under replicated queue. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9ec3d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9ec3d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9ec3d78
Branch: refs/heads/HADOOP-15461
Commit: e9ec3d78f520a8543dc77d763d4b358aa608bae8
Parents: 56a4cdb
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:35:30 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:35:30 2018 -0700
----------------------------------------------------------------------
.../container/replication/ReplicationQueue.java | 76 +++++++++++
.../replication/ReplicationRequest.java | 106 +++++++++++++++
.../container/replication/package-info.java | 23 ++++
.../replication/TestReplicationQueue.java | 134 +++++++++++++++++++
.../container/replication/package-info.java | 23 ++++
5 files changed, 362 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
new file mode 100644
index 0000000..e0a2351
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+
+/**
+ * Priority queue to handle under-replicated and over replicated containers
+ * in ozone. ReplicationManager will consume these messages and decide
+ * accordingly.
+ */
+public class ReplicationQueue {
+
+ private final Queue<ReplicationRequest> queue;
+
+ ReplicationQueue() {
+ queue = new PriorityQueue<>();
+ }
+
+ public synchronized boolean add(ReplicationRequest repObj) {
+ if (this.queue.contains(repObj)) {
+ // Remove the earlier message and insert this one
+ this.queue.remove(repObj);
+ }
+ return this.queue.add(repObj);
+ }
+
+ public synchronized boolean remove(ReplicationRequest repObj) {
+ return queue.remove(repObj);
+ }
+
+ /**
+ * Retrieves, but does not remove, the head of this queue,
+ * or returns {@code null} if this queue is empty.
+ *
+ * @return the head of this queue, or {@code null} if this queue is empty
+ */
+ public synchronized ReplicationRequest peek() {
+ return queue.peek();
+ }
+
+ /**
+ * Retrieves and removes the head of this queue,
+ * or returns {@code null} if this queue is empty.
+ *
+ * @return the head of this queue, or {@code null} if this queue is empty
+ */
+ public synchronized ReplicationRequest poll() {
+ return queue.poll();
+ }
+
+ public synchronized boolean removeAll(List<ReplicationRequest> repObjs) {
+ return queue.removeAll(repObjs);
+ }
+
+ public int size() {
+ return queue.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
new file mode 100644
index 0000000..a6ccce1
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationRequest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.Serializable;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+/**
+ * Wrapper class for hdds replication queue. Implements its natural
+ * ordering for priority queue.
+ */
+public class ReplicationRequest implements Comparable<ReplicationRequest>,
+ Serializable {
+ private final long containerId;
+ private final short replicationCount;
+ private final short expecReplicationCount;
+ private final long timestamp;
+
+ public ReplicationRequest(long containerId, short replicationCount,
+ long timestamp, short expecReplicationCount) {
+ this.containerId = containerId;
+ this.replicationCount = replicationCount;
+ this.timestamp = timestamp;
+ this.expecReplicationCount = expecReplicationCount;
+ }
+
+ /**
+ * Compares this object with the specified object for order. Returns a
+ * negative integer, zero, or a positive integer as this object is less
+ * than, equal to, or greater than the specified object.
+ * @param o the object to be compared.
+ * @return a negative integer, zero, or a positive integer as this object
+ * is less than, equal to, or greater than the specified object.
+ * @throws NullPointerException if the specified object is null
+ * @throws ClassCastException if the specified object's type prevents it
+ * from being compared to this object.
+ */
+ @Override
+ public int compareTo(ReplicationRequest o) {
+ if (o == null) {
+ return 1;
+ }
+ if (this == o) {
+ return 0;
+ }
+ int retVal = Integer
+ .compare(getReplicationCount() - getExpecReplicationCount(),
+ o.getReplicationCount() - o.getExpecReplicationCount());
+ if (retVal != 0) {
+ return retVal;
+ }
+ return Long.compare(getTimestamp(), o.getTimestamp());
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(91, 1011)
+ .append(getContainerId())
+ .toHashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationRequest that = (ReplicationRequest) o;
+ return new EqualsBuilder().append(getContainerId(), that.getContainerId())
+ .isEquals();
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ public short getReplicationCount() {
+ return replicationCount;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public short getExpecReplicationCount() {
+ return expecReplicationCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..7f335e3
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Ozone Container replicaton related classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
new file mode 100644
index 0000000..6d74c68
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.Random;
+import java.util.UUID;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for ReplicationQueue.
+ */
+public class TestReplicationQueue {
+
+ private ReplicationQueue replicationQueue;
+ private Random random;
+
+ @Before
+ public void setUp() {
+ replicationQueue = new ReplicationQueue();
+ random = new Random();
+ }
+
+ @Test
+ public void testDuplicateAddOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationRequest obj1, obj2, obj3;
+ long time = Time.monotonicNow();
+ obj1 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
+ obj2 = new ReplicationRequest(contId, (short) 2, time + 1, (short) 3);
+ obj3 = new ReplicationRequest(contId, (short) 1, time+2, (short) 3);
+
+ replicationQueue.add(obj1);
+ replicationQueue.add(obj2);
+ replicationQueue.add(obj3);
+ Assert.assertEquals("Should add only 1 msg as second one is duplicate",
+ 1, replicationQueue.size());
+ ReplicationRequest temp = replicationQueue.poll();
+ Assert.assertEquals(temp, obj3);
+ }
+
+ @Test
+ public void testPollOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationRequest msg1, msg2, msg3, msg4, msg5;
+ msg1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
+ (short) 3);
+ long time = Time.monotonicNow();
+ msg2 = new ReplicationRequest(contId + 1, (short) 4, time, (short) 3);
+ msg3 = new ReplicationRequest(contId + 2, (short) 0, time, (short) 3);
+ msg4 = new ReplicationRequest(contId, (short) 2, time, (short) 3);
+ // Replication message for same container but different nodeId
+ msg5 = new ReplicationRequest(contId + 1, (short) 2, time, (short) 3);
+
+ replicationQueue.add(msg1);
+ replicationQueue.add(msg2);
+ replicationQueue.add(msg3);
+ replicationQueue.add(msg4);
+ replicationQueue.add(msg5);
+ Assert.assertEquals("Should have 3 objects",
+ 3, replicationQueue.size());
+
+ // Since Priority queue orders messages according to replication count,
+ // message with lowest replication should be first
+ ReplicationRequest temp;
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 2 objects",
+ 2, replicationQueue.size());
+ Assert.assertEquals(temp, msg3);
+
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 1 objects",
+ 1, replicationQueue.size());
+ Assert.assertEquals(temp, msg5);
+
+ // Message 2 should be ordered before message 5 as both have same replication
+ // number but message 2 has earlier timestamp.
+ temp = replicationQueue.poll();
+ Assert.assertEquals("Should have 0 objects",
+ replicationQueue.size(), 0);
+ Assert.assertEquals(temp, msg4);
+ }
+
+ @Test
+ public void testRemoveOp() {
+ long contId = random.nextLong();
+ String nodeId = UUID.randomUUID().toString();
+ ReplicationRequest obj1, obj2, obj3;
+ obj1 = new ReplicationRequest(contId, (short) 1, Time.monotonicNow(),
+ (short) 3);
+ obj2 = new ReplicationRequest(contId + 1, (short) 2, Time.monotonicNow(),
+ (short) 3);
+ obj3 = new ReplicationRequest(contId + 2, (short) 3, Time.monotonicNow(),
+ (short) 3);
+
+ replicationQueue.add(obj1);
+ replicationQueue.add(obj2);
+ replicationQueue.add(obj3);
+ Assert.assertEquals("Should have 3 objects",
+ 3, replicationQueue.size());
+
+ replicationQueue.remove(obj3);
+ Assert.assertEquals("Should have 2 objects",
+ 2, replicationQueue.size());
+
+ replicationQueue.remove(obj2);
+ Assert.assertEquals("Should have 1 objects",
+ 1, replicationQueue.size());
+
+ replicationQueue.remove(obj1);
+ Assert.assertEquals("Should have 0 objects",
+ 0, replicationQueue.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9ec3d78/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5b1fd0f
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * SCM Testing and Mocking Utils.
+ */
+package org.apache.hadoop.ozone.container.replication;
+// Test classes for Replication functionality.
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager
and node pool handling from SCM. Contributed by Elek Marton
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf03cc4/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index 2bd43fb..edc0d7b 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.scm.cli;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
@@ -60,13 +59,11 @@ import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
/**
@@ -111,15 +108,6 @@ public class SQLCLI extends Configured implements Tool {
private static final String INSERT_CONTAINER_MEMBERS =
"INSERT INTO containerMembers (containerName, datanodeUUID) " +
"VALUES (\"%s\", \"%s\")";
- // for nodepool.db
- private static final String CREATE_NODE_POOL =
- "CREATE TABLE nodePool (" +
- "datanodeUUID TEXT NOT NULL," +
- "poolName TEXT NOT NULL," +
- "PRIMARY KEY(datanodeUUID, poolName))";
- private static final String INSERT_NODE_POOL =
- "INSERT INTO nodePool (datanodeUUID, poolName) " +
- "VALUES (\"%s\", \"%s\")";
// and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
// for openContainer.db
private static final String CREATE_OPEN_CONTAINER =
@@ -285,9 +273,6 @@ public class SQLCLI extends Configured implements Tool {
if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
LOG.info("Converting container DB");
convertContainerDB(dbPath, outPath);
- } else if (dbName.toString().equals(NODEPOOL_DB)) {
- LOG.info("Converting node pool DB");
- convertNodePoolDB(dbPath, outPath);
} else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
LOG.info("Converting open container DB");
convertOpenContainerDB(dbPath, outPath);
@@ -543,66 +528,7 @@ public class SQLCLI extends Configured implements Tool {
}
LOG.info("Insertion completed.");
}
- /**
- * Converts nodePool.db to sqlite. The schema of sql db:
- * two tables, nodePool and datanodeInfo (the same datanode Info as for
- * container.db).
- *
- * nodePool
- * ---------------------------------------------------------
- * datanodeUUID* | poolName*
- * ---------------------------------------------------------
- *
- * datanodeInfo:
- * ---------------------------------------------------------
- * hostname | datanodeUUid* | xferPort | ipcPort
- * ---------------------------------------------------------
- *
- * --------------------------------
- * |containerPort
- * --------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create table for sql node pool db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_NODE_POOL);
- executeSQL(conn, CREATE_DATANODE_INFO);
- dbStore.iterate(null, (key, value) -> {
- DatanodeDetails nodeId = DatanodeDetails
- .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
- .PARSER.parseFrom(key));
- String blockPool = DFSUtil.bytes2String(value);
- try {
- insertNodePoolDB(conn, blockPool, nodeId);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- private void insertNodePoolDB(Connection conn, String blockPool,
- DatanodeDetails datanodeDetails) throws SQLException {
- String insertNodePool = String.format(INSERT_NODE_POOL,
- datanodeDetails.getUuidString(), blockPool);
- executeSQL(conn, insertNodePool);
-
- String insertDatanodeDetails = String
- .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
- datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
- datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
- .getValue());
- executeSQL(conn, insertDatanodeDetails);
- }
/**
* Convert openContainer.db to sqlite db file. This is rather simple db,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-15458.
TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed
by Xiao Liang.
Posted by gi...@apache.org.
HADOOP-15458. TestLocalFileSystem#testFSOutputStreamBuilder fails on Windows. Contributed by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abc3e4ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abc3e4ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abc3e4ba
Branch: refs/heads/HADOOP-15461
Commit: abc3e4bad905efde5a4881e8a072c68f6e910ade
Parents: 1ba4e62
Author: Inigo Goiri <in...@apache.org>
Authored: Mon Jun 25 09:50:27 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Mon Jun 25 09:50:27 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/TestLocalFileSystem.java | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/abc3e4ba/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
index 0e337b4..d5622af 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java
@@ -689,17 +689,18 @@ public class TestLocalFileSystem {
// and permission
FSDataOutputStreamBuilder builder =
fileSys.createFile(path);
- builder.build();
- Assert.assertEquals("Should be default block size",
- builder.getBlockSize(), fileSys.getDefaultBlockSize());
- Assert.assertEquals("Should be default replication factor",
- builder.getReplication(), fileSys.getDefaultReplication());
- Assert.assertEquals("Should be default buffer size",
- builder.getBufferSize(),
- fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
- IO_FILE_BUFFER_SIZE_DEFAULT));
- Assert.assertEquals("Should be default permission",
- builder.getPermission(), FsPermission.getFileDefault());
+ try (FSDataOutputStream stream = builder.build()) {
+ Assert.assertEquals("Should be default block size",
+ builder.getBlockSize(), fileSys.getDefaultBlockSize());
+ Assert.assertEquals("Should be default replication factor",
+ builder.getReplication(), fileSys.getDefaultReplication());
+ Assert.assertEquals("Should be default buffer size",
+ builder.getBufferSize(),
+ fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
+ IO_FILE_BUFFER_SIZE_DEFAULT));
+ Assert.assertEquals("Should be default permission",
+ builder.getPermission(), FsPermission.getFileDefault());
+ }
// Test set 0 to replication, block size and buffer size
builder = fileSys.createFile(path);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HDDS-184. Upgrade common-langs
version to 3.7 in hadoop-tools/hadoop-ozone. Contributed by Takanobu Asanuma.
Posted by gi...@apache.org.
HDDS-184. Upgrade common-langs version to 3.7 in hadoop-tools/hadoop-ozone.
Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca14fec0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca14fec0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca14fec0
Branch: refs/heads/HADOOP-15461
Commit: ca14fec02fb14e1b708f266bc715e84ae9784d6f
Parents: 8a32bc3
Author: Anu Engineer <ae...@apache.org>
Authored: Sat Jun 23 15:49:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sat Jun 23 15:49:44 2018 -0700
----------------------------------------------------------------------
.../ozone/genconf/TestGenerateOzoneRequiredConfigurations.java | 2 +-
.../src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +-
.../java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
index cfd1159..c2f5eb7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.ozone.genconf;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.hamcrest.CoreMatchers;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 0ff1d50..6906a9d 100644
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index b82c4a1..ad21f28 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.ozone;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
index 5a7cb4f..a225702 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -31,7 +31,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca14fec0/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index 176b614..8417e46 100644
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.ozone.contract;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.FileSystem;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-8438.
TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.
Posted by gi...@apache.org.
YARN-8438. TestContainer.testKillOnNew flaky on trunk. Contributed by Szilard Nemeth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35ec9401
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35ec9401
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35ec9401
Branch: refs/heads/HADOOP-15461
Commit: 35ec9401e829bfa10994790659a26b0babacae35
Parents: 7a3c6e9
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Mon Jun 25 09:23:11 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Mon Jun 25 15:47:54 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/TestContainer.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35ec9401/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 1a263ee..edf26d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -116,7 +116,7 @@ public class TestContainer {
final NodeManagerMetrics metrics = NodeManagerMetrics.create();
final Configuration conf = new YarnConfiguration();
final String FAKE_LOCALIZATION_ERROR = "Fake localization error";
-
+
/**
* Verify correct container request events sent to localizer.
*/
@@ -591,9 +591,8 @@ public class TestContainer {
Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
containerMetrics.exitCode.value());
Assert.assertTrue(containerMetrics.startTime.value() > 0);
- Assert.assertTrue(
- containerMetrics.finishTime.value() > containerMetrics.startTime
- .value());
+ Assert.assertTrue(containerMetrics.finishTime.value() >=
+ containerMetrics.startTime.value());
Assert.assertEquals(ContainerEventType.KILL_CONTAINER,
wc.initStateToEvent.get(ContainerState.NEW));
Assert.assertEquals(ContainerState.DONE,
@@ -1612,4 +1611,5 @@ public class TestContainer {
return ((ContainerImpl)c).getRetryPolicy();
}
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-8401. [UI2] new ui is not
accessible with out internet connection. Contributed by Bibin A Chundatt.
Posted by gi...@apache.org.
YARN-8401. [UI2] new ui is not accessible with out internet connection. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbaff369
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbaff369
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbaff369
Branch: refs/heads/HADOOP-15461
Commit: fbaff369e9b9022723a7b2c6f25e71122a8f8a15
Parents: bedc4fe
Author: Sunil G <su...@apache.org>
Authored: Wed Jun 27 10:35:15 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Wed Jun 27 10:35:15 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbaff369/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
index ddb8532..ac74d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/WEB-INF/web.xml
@@ -16,10 +16,6 @@
* limitations under the License.
-->
-<!DOCTYPE web-app PUBLIC
- "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
- "http://java.sun.com/dtd/web-app_2_3.dtd" >
-
-<web-app>
+<web-app version="2.4" xmlns="http://java.sun.com/xml/ns/j2ee">
<display-name>YARN UI</display-name>
</web-app>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-178: DN should update
transactionId on block delete. Contributed by Lokesh Jain.
Posted by gi...@apache.org.
HDDS-178: DN should update transactionId on block delete. Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4d7227a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4d7227a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4d7227a
Branch: refs/heads/HADOOP-15461
Commit: e4d7227aad586f055b47bdc90c65361f9fb23146
Parents: 852ca6b
Author: Nanda kumar <na...@apache.org>
Authored: Fri Jun 29 04:37:16 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Jun 29 04:37:16 2018 +0530
----------------------------------------------------------------------
.../common/impl/ContainerManagerImpl.java | 7 +-
.../common/interfaces/ContainerManager.java | 2 +
.../DeleteBlocksCommandHandler.java | 29 ++-
.../commandhandler/TestBlockDeletion.java | 211 +++++++++++++++++++
4 files changed, 243 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
index 02572a8..e81f1c6 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
@@ -1111,5 +1111,10 @@ public class ContainerManagerImpl implements ContainerManager {
return cData.getKeyCount();
}
-
+ @Override
+ public void updateDeleteTransactionId(long containerId,
+ long deleteTransactionId) {
+ containerMap.get(containerId)
+ .updateDeleteTransactionId(deleteTransactionId);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
index 49b68dc..cf68b08 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
@@ -264,4 +264,6 @@ public interface ContainerManager extends RwLock {
*/
long getNumKeys(long containerId);
+ void updateDeleteTransactionId(long containerId, long deleteTransactionId);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index f954d98..d215da9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -158,6 +158,13 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
containerInfo.getDBPath());
}
+ if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) {
+ LOG.debug(String.format("Ignoring delete blocks for containerId: %d."
+ + " Outdated delete transactionId %d < %d", containerId,
+ delTX.getTxID(), containerInfo.getDeleteTransactionId()));
+ return;
+ }
+
int newDeletionBlocks = 0;
MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
for (Long blk : delTX.getLocalIDList()) {
@@ -165,10 +172,20 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
byte[] blkBytes = Longs.toByteArray(blk);
byte[] blkInfo = containerDB.get(blkBytes);
if (blkInfo != null) {
+ byte[] deletingKeyBytes =
+ DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk);
+ byte[] deletedKeyBytes =
+ DFSUtil.string2Bytes(OzoneConsts.DELETED_KEY_PREFIX + blk);
+ if (containerDB.get(deletingKeyBytes) != null
+ || containerDB.get(deletedKeyBytes) != null) {
+ LOG.debug(String.format(
+ "Ignoring delete for block %d in container %d."
+ + " Entry already added.", blk, containerId));
+ continue;
+ }
// Found the block in container db,
// use an atomic update to change its state to deleting.
- batch.put(DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk),
- blkInfo);
+ batch.put(deletingKeyBytes, blkInfo);
batch.delete(blkBytes);
try {
containerDB.writeBatch(batch);
@@ -186,11 +203,13 @@ public class DeleteBlocksCommandHandler implements CommandHandler {
LOG.debug("Block {} not found or already under deletion in"
+ " container {}, skip deleting it.", blk, containerId);
}
- containerDB.put(DFSUtil.string2Bytes(
- OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
- Longs.toByteArray(delTX.getTxID()));
}
+ containerDB.put(DFSUtil.string2Bytes(
+ OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()),
+ Longs.toByteArray(delTX.getTxID()));
+ containerManager
+ .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID());
// update pending deletion blocks count in in-memory container status
containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d7227a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
new file mode 100644
index 0000000..43e3f50
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
+import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
+import org.apache.hadoop.ozone.ksm.KeySpaceManager;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
+import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.utils.MetadataStore;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+
+public class TestBlockDeletion {
+ private static OzoneConfiguration conf = null;
+ private static ObjectStore store;
+ private static ContainerManagerImpl dnContainerManager = null;
+ private static StorageContainerManager scm = null;
+ private static KeySpaceManager ksm = null;
+ private static Set<Long> containerIdsWithDeletedBlocks;
+
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+
+ String path =
+ GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName());
+ File baseDir = new File(path);
+ baseDir.mkdirs();
+
+ path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
+ OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
+
+ conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
+ conf.setQuietMode(false);
+ conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
+
+ MiniOzoneCluster cluster =
+ MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+ cluster.waitForClusterToBeReady();
+ store = OzoneClientFactory.getRpcClient(conf).getObjectStore();
+ dnContainerManager =
+ (ContainerManagerImpl) cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine().getContainer().getContainerManager();
+ ksm = cluster.getKeySpaceManager();
+ scm = cluster.getStorageContainerManager();
+ containerIdsWithDeletedBlocks = new HashSet<>();
+ }
+
+ @Test(timeout = 60000)
+ public void testBlockDeletion()
+ throws IOException, InterruptedException {
+ String volumeName = UUID.randomUUID().toString();
+ String bucketName = UUID.randomUUID().toString();
+
+ String value = RandomStringUtils.random(1000000);
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ String keyName = UUID.randomUUID().toString();
+
+ OzoneOutputStream out = bucket.createKey(keyName, value.getBytes().length,
+ ReplicationType.STAND_ALONE, ReplicationFactor.ONE);
+ out.write(value.getBytes());
+ out.close();
+
+ KsmKeyArgs keyArgs = new KsmKeyArgs.Builder().setVolumeName(volumeName)
+ .setBucketName(bucketName).setKeyName(keyName).setDataSize(0)
+ .setType(HddsProtos.ReplicationType.STAND_ALONE)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).build();
+ List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroupList =
+ ksm.lookupKey(keyArgs).getKeyLocationVersions();
+
+ // verify key blocks were created in DN.
+ Assert.assertTrue(verifyBlocksCreated(ksmKeyLocationInfoGroupList));
+ // No containers with deleted blocks
+ Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty());
+ // Delete transactionIds for the containers should be 0
+ matchContainerTransactionIds();
+ ksm.deleteKey(keyArgs);
+ Thread.sleep(5000);
+ // The blocks should be deleted in the DN.
+ Assert.assertTrue(verifyBlocksDeleted(ksmKeyLocationInfoGroupList));
+
+ // Few containers with deleted blocks
+ Assert.assertTrue(!containerIdsWithDeletedBlocks.isEmpty());
+ // Containers in the DN and SCM should have same delete transactionIds
+ matchContainerTransactionIds();
+ }
+
+ private void matchContainerTransactionIds() throws IOException {
+ List<ContainerData> containerDataList = new ArrayList<>();
+ dnContainerManager.listContainer(0, 10000, containerDataList);
+ for (ContainerData containerData : containerDataList) {
+ long containerId = containerData.getContainerID();
+ if (containerIdsWithDeletedBlocks.contains(containerId)) {
+ Assert.assertTrue(
+ scm.getContainerInfo(containerId).getDeleteTransactionId() > 0);
+ } else {
+ Assert.assertEquals(
+ scm.getContainerInfo(containerId).getDeleteTransactionId(), 0);
+ }
+ Assert.assertEquals(dnContainerManager.readContainer(containerId)
+ .getDeleteTransactionId(),
+ scm.getContainerInfo(containerId).getDeleteTransactionId());
+ }
+ }
+
+ private boolean verifyBlocksCreated(
+ List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+ throws IOException {
+ return performOperationOnKeyContainers((blockID) -> {
+ try {
+ MetadataStore db = KeyUtils.getDB(
+ dnContainerManager.getContainerMap().get(blockID.getContainerID()),
+ conf);
+ Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID())));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }, ksmKeyLocationInfoGroups);
+ }
+
+ private boolean verifyBlocksDeleted(
+ List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+ throws IOException {
+ return performOperationOnKeyContainers((blockID) -> {
+ try {
+ MetadataStore db = KeyUtils.getDB(
+ dnContainerManager.getContainerMap().get(blockID.getContainerID()),
+ conf);
+ Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID())));
+ Assert.assertNull(db.get(DFSUtil.string2Bytes(
+ OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID())));
+ Assert.assertNotNull(DFSUtil.string2Bytes(
+ OzoneConsts.DELETED_KEY_PREFIX + blockID.getLocalID()));
+ containerIdsWithDeletedBlocks.add(blockID.getContainerID());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }, ksmKeyLocationInfoGroups);
+ }
+
+ private boolean performOperationOnKeyContainers(Consumer<BlockID> consumer,
+ List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
+ throws IOException {
+
+ try {
+ for (KsmKeyLocationInfoGroup ksmKeyLocationInfoGroup : ksmKeyLocationInfoGroups) {
+ List<KsmKeyLocationInfo> ksmKeyLocationInfos =
+ ksmKeyLocationInfoGroup.getLocationList();
+ for (KsmKeyLocationInfo ksmKeyLocationInfo : ksmKeyLocationInfos) {
+ BlockID blockID = ksmKeyLocationInfo.getBlockID();
+ consumer.accept(blockID);
+ }
+ }
+ } catch (Error e) {
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-8451. Multiple NM heartbeat
thread created when a slow NM resync with RM. Contributed by Botong Huang
Posted by gi...@apache.org.
YARN-8451. Multiple NM heartbeat thread created when a slow NM resync with RM. Contributed by Botong Huang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10047014
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10047014
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10047014
Branch: refs/heads/HADOOP-15461
Commit: 100470140d86eede0fa240a9aa93226f274ee4f5
Parents: a820738
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jun 29 13:06:28 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jun 29 13:06:28 2018 -0500
----------------------------------------------------------------------
.../yarn/server/nodemanager/NodeManager.java | 66 +++++++++++++-------
.../nodemanager/TestNodeManagerResync.java | 56 +++++++++++++++++
2 files changed, 98 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10047014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 2748a8f..c8234bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -128,6 +128,7 @@ public class NodeManager extends CompositeService
// the NM collector service is set only if the timeline service v.2 is enabled
private NMCollectorService nmCollectorService;
private NodeStatusUpdater nodeStatusUpdater;
+ private AtomicBoolean resyncingWithRM = new AtomicBoolean(false);
private NodeResourceMonitor nodeResourceMonitor;
private static CompositeServiceShutdownHook nodeManagerShutdownHook;
private NMStateStoreService nmStore = null;
@@ -393,7 +394,7 @@ public class NodeManager extends CompositeService
addService(del);
// NodeManager level dispatcher
- this.dispatcher = new AsyncDispatcher("NM Event dispatcher");
+ this.dispatcher = createNMDispatcher();
nodeHealthChecker =
new NodeHealthCheckerService(
@@ -517,31 +518,41 @@ public class NodeManager extends CompositeService
}
protected void resyncWithRM() {
- //we do not want to block dispatcher thread here
- new Thread() {
- @Override
- public void run() {
- try {
- if (!rmWorkPreservingRestartEnabled) {
- LOG.info("Cleaning up running containers on resync");
- containerManager.cleanupContainersOnNMResync();
- // Clear all known collectors for resync.
- if (context.getKnownCollectors() != null) {
- context.getKnownCollectors().clear();
+ // Create a thread for resync because we do not want to block dispatcher
+ // thread here. Also use locking to make sure only one thread is running at
+ // a time.
+ if (this.resyncingWithRM.getAndSet(true)) {
+ // Some other thread is already created for resyncing, do nothing
+ } else {
+ // We have got the lock, create a new thread
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ if (!rmWorkPreservingRestartEnabled) {
+ LOG.info("Cleaning up running containers on resync");
+ containerManager.cleanupContainersOnNMResync();
+ // Clear all known collectors for resync.
+ if (context.getKnownCollectors() != null) {
+ context.getKnownCollectors().clear();
+ }
+ } else {
+ LOG.info("Preserving containers on resync");
+ // Re-register known timeline collectors.
+ reregisterCollectors();
}
- } else {
- LOG.info("Preserving containers on resync");
- // Re-register known timeline collectors.
- reregisterCollectors();
+ ((NodeStatusUpdaterImpl) nodeStatusUpdater)
+ .rebootNodeStatusUpdaterAndRegisterWithRM();
+ } catch (YarnRuntimeException e) {
+ LOG.error("Error while rebooting NodeStatusUpdater.", e);
+ shutDown(NodeManagerStatus.EXCEPTION.getExitCode());
+ } finally {
+ // Release lock
+ resyncingWithRM.set(false);
}
- ((NodeStatusUpdaterImpl) nodeStatusUpdater)
- .rebootNodeStatusUpdaterAndRegisterWithRM();
- } catch (YarnRuntimeException e) {
- LOG.error("Error while rebooting NodeStatusUpdater.", e);
- shutDown(NodeManagerStatus.EXCEPTION.getExitCode());
}
- }
- }.start();
+ }.start();
+ }
}
/**
@@ -946,7 +957,14 @@ public class NodeManager extends CompositeService
ContainerManagerImpl getContainerManager() {
return containerManager;
}
-
+
+ /**
+ * Unit test friendly.
+ */
+ protected AsyncDispatcher createNMDispatcher() {
+ return new AsyncDispatcher("NM Event dispatcher");
+ }
+
//For testing
Dispatcher getNMDispatcher(){
return dispatcher;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10047014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index cf33775..b3f4e1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -37,6 +37,7 @@ import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -64,7 +65,9 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -107,6 +110,7 @@ public class TestNodeManagerResync {
private FileContext localFS;
private CyclicBarrier syncBarrier;
private CyclicBarrier updateBarrier;
+ private AtomicInteger resyncThreadCount;
private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false);
private final NodeManagerEvent resyncEvent =
@@ -125,6 +129,7 @@ public class TestNodeManagerResync {
nmLocalDir.mkdirs();
syncBarrier = new CyclicBarrier(2);
updateBarrier = new CyclicBarrier(2);
+ resyncThreadCount = new AtomicInteger(0);
}
@After
@@ -186,6 +191,41 @@ public class TestNodeManagerResync {
}
@SuppressWarnings("resource")
+ @Test(timeout = 30000)
+ public void testNMMultipleResyncEvent()
+ throws IOException, InterruptedException {
+ TestNodeManager1 nm = new TestNodeManager1(false);
+ YarnConfiguration conf = createNMConfig();
+
+ int resyncEventCount = 4;
+ try {
+ nm.init(conf);
+ nm.start();
+ Assert.assertEquals(1, nm.getNMRegistrationCount());
+ for (int i = 0; i < resyncEventCount; i++) {
+ nm.getNMDispatcher().getEventHandler().handle(resyncEvent);
+ }
+
+ DrainDispatcher dispatcher = (DrainDispatcher) nm.getNMDispatcher();
+ dispatcher.await();
+ LOG.info("NM dispatcher drained");
+
+ // Wait for the resync thread to finish
+ try {
+ syncBarrier.await();
+ } catch (BrokenBarrierException e) {
+ }
+ LOG.info("Barrier wait done for the resync thread");
+
+ // Resync should only happen once
+ Assert.assertEquals(2, nm.getNMRegistrationCount());
+ Assert.assertFalse("NM shutdown called.", isNMShutdownCalled.get());
+ } finally {
+ nm.stop();
+ }
+ }
+
+ @SuppressWarnings("resource")
@Test(timeout=10000)
public void testNMshutdownWhenResyncThrowException() throws IOException,
InterruptedException, YarnException {
@@ -400,6 +440,11 @@ public class TestNodeManagerResync {
}
@Override
+ protected AsyncDispatcher createNMDispatcher() {
+ return new DrainDispatcher();
+ }
+
+ @Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new TestNodeStatusUpdaterImpl1(context, dispatcher,
@@ -410,6 +455,14 @@ public class TestNodeManagerResync {
return registrationCount;
}
+ @Override
+ protected void shutDown(int exitCode) {
+ synchronized (isNMShutdownCalled) {
+ isNMShutdownCalled.set(true);
+ isNMShutdownCalled.notify();
+ }
+ }
+
class TestNodeStatusUpdaterImpl1 extends MockNodeStatusUpdater {
public TestNodeStatusUpdaterImpl1(Context context, Dispatcher dispatcher,
@@ -428,6 +481,9 @@ public class TestNodeManagerResync {
ConcurrentMap<ContainerId, org.apache.hadoop.yarn.server.nodemanager
.containermanager.container.Container> containers =
getNMContext().getContainers();
+ if (resyncThreadCount.incrementAndGet() > 1) {
+ throw new YarnRuntimeException("Multiple resync thread created!");
+ }
try {
try {
if (containersShouldBePreserved) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-8455. Add basic ACL check for
all ATS v2 REST APIs. Contributed by Rohith Sharma K S.
Posted by gi...@apache.org.
YARN-8455. Add basic ACL check for all ATS v2 REST APIs. Contributed by Rohith Sharma K S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/469b29c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/469b29c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/469b29c0
Branch: refs/heads/HADOOP-15461
Commit: 469b29c0817b7bf1902c9195c4f8d031a909e1c9
Parents: 73746c5
Author: Sunil G <su...@apache.org>
Authored: Fri Jun 29 10:02:53 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Fri Jun 29 10:02:53 2018 -0700
----------------------------------------------------------------------
.../reader/TimelineFromIdConverter.java | 93 +++++++++
.../reader/TimelineReaderWebServices.java | 198 +++++++++++++++----
.../TestTimelineReaderWebServicesBasicAcl.java | 154 +++++++++++++++
3 files changed, 407 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
new file mode 100644
index 0000000..5f5f0b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineFromIdConverter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.List;
+
+/**
+ * Used for decoding FROM_ID
+ */
+enum TimelineFromIdConverter {
+
+ APPLICATION_FROMID {
+ @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+ if (fromId == null) {
+ return null;
+ }
+
+ List<String> appTupleList = TimelineReaderUtils.split(fromId);
+ if (appTupleList == null || appTupleList.size() != 5) {
+ throw new IllegalArgumentException(
+ "Invalid row key for application table.");
+ }
+
+ return new TimelineReaderContext(appTupleList.get(0), appTupleList.get(1),
+ appTupleList.get(2), Long.parseLong(appTupleList.get(3)),
+ appTupleList.get(4), null, null);
+ }
+ },
+
+ SUB_APPLICATION_ENTITY_FROMID {
+ @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+ if (fromId == null) {
+ return null;
+ }
+ List<String> split = TimelineReaderUtils.split(fromId);
+ if (split == null || split.size() != 6) {
+ throw new IllegalArgumentException(
+ "Invalid row key for sub app table.");
+ }
+
+ String subAppUserId = split.get(0);
+ String clusterId = split.get(1);
+ String entityType = split.get(2);
+ Long entityIdPrefix = Long.valueOf(split.get(3));
+ String entityId = split.get(4);
+ String userId = split.get(5);
+ return new TimelineReaderContext(clusterId, userId, null, null, null,
+ entityType, entityIdPrefix, entityId, subAppUserId);
+ }
+ },
+
+ GENERIC_ENTITY_FROMID {
+ @Override TimelineReaderContext decodeUID(String fromId) throws Exception {
+ if (fromId == null) {
+ return null;
+ }
+ List<String> split = TimelineReaderUtils.split(fromId);
+ if (split == null || split.size() != 8) {
+ throw new IllegalArgumentException("Invalid row key for entity table.");
+ }
+ Long flowRunId = Long.valueOf(split.get(3));
+ Long entityIdPrefix = Long.valueOf(split.get(6));
+ return new TimelineReaderContext(split.get(0), split.get(1), split.get(2),
+ flowRunId, split.get(4), split.get(5), entityIdPrefix, split.get(7));
+ }
+ };
+
+ /**
+ * Decodes FROM_ID depending on FROM_ID implementation.
+ *
+ * @param fromId FROM_ID to be decoded.
+ * @return a {@link TimelineReaderContext} object if FROM_ID passed can be
+ * decoded, null otherwise.
+ * @throws Exception if any problem occurs while decoding.
+ */
+ abstract TimelineReaderContext decodeUID(String fromId) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 7bf66b0..7f96bfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.common.annotations.VisibleForTesting;
@@ -188,6 +189,8 @@ public class TimelineReaderWebServices {
"Filter Parsing failed." : e.getMessage());
} else if (e instanceof BadRequestException) {
throw (BadRequestException)e;
+ } else if (e instanceof ForbiddenException) {
+ throw (ForbiddenException) e;
} else {
LOG.error("Error while processing REST request", e);
throw new WebApplicationException(e,
@@ -339,6 +342,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForGenericEntities(entities, callerUGI, entityType);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -607,13 +611,15 @@ public class TimelineReaderWebServices {
.createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
appId, entityType, null, null);
entities = timelineReaderManager.getEntities(context,
- TimelineReaderWebServicesUtils.createTimelineEntityFilters(
- limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
- infofilters, conffilters, metricfilters, eventfilters,
- fromId),
- TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
- metricsTimeStart, metricsTimeEnd));
+ TimelineReaderWebServicesUtils
+ .createTimelineEntityFilters(limit, createdTimeStart,
+ createdTimeEnd, relatesTo, isRelatedTo, infofilters,
+ conffilters, metricfilters, eventfilters, fromId),
+ TimelineReaderWebServicesUtils
+ .createTimelineDataToRetrieve(confsToRetrieve, metricsToRetrieve,
+ fields, metricsLimit, metricsTimeStart, metricsTimeEnd));
+
+ checkAccessForGenericEntities(entities, callerUGI, entityType);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -704,6 +710,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForGenericEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -893,6 +900,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForGenericEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -956,6 +964,8 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
+ // TODO to be removed or modified once ACL story is played
+ checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
@@ -1063,12 +1073,16 @@ public class TimelineReaderWebServices {
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
try {
- entity = timelineReaderManager.getEntity(
- TimelineReaderWebServicesUtils.createTimelineReaderContext(
- clusterId, userId, flowName, flowRunId, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
- TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, metricsToRetrieve, null, null, null, null));
+ TimelineReaderContext context = TimelineReaderWebServicesUtils
+ .createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
+ null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ // TODO to be removed or modified once ACL story is played
+ checkAccess(timelineReaderManager, callerUGI, context.getUserId());
+
+ entity = timelineReaderManager.getEntity(context,
+ TimelineReaderWebServicesUtils
+ .createTimelineDataToRetrieve(null, metricsToRetrieve, null, null,
+ null, null));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1156,6 +1170,8 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
+ // TODO to be removed or modified once ACL story is played
+ checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@@ -1304,15 +1320,21 @@ public class TimelineReaderWebServices {
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
try {
- entities = timelineReaderManager.getEntities(
- TimelineReaderWebServicesUtils.createTimelineReaderContext(
- clusterId, userId, flowName, null, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null, null),
- TimelineReaderWebServicesUtils.createTimelineEntityFilters(
- limit, createdTimeStart, createdTimeEnd, null, null, null,
- null, null, null, fromId),
- TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, metricsToRetrieve, fields, null, null, null));
+ TimelineReaderContext timelineReaderContext = TimelineReaderWebServicesUtils
+ .createTimelineReaderContext(clusterId, userId, flowName, null,
+ null, TimelineEntityType.YARN_FLOW_RUN.toString(), null,
+ null);
+ // TODO to be removed or modified once ACL story is played
+ checkAccess(timelineReaderManager, callerUGI,
+ timelineReaderContext.getUserId());
+
+ entities = timelineReaderManager.getEntities(timelineReaderContext,
+ TimelineReaderWebServicesUtils
+ .createTimelineEntityFilters(limit, createdTimeStart,
+ createdTimeEnd, null, null, null, null, null, null, fromId),
+ TimelineReaderWebServicesUtils
+ .createTimelineDataToRetrieve(null, metricsToRetrieve, fields,
+ null, null, null));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or fromId");
@@ -1435,7 +1457,6 @@ public class TimelineReaderWebServices {
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
- Configuration config = timelineReaderManager.getConfig();
Set<TimelineEntity> entities = null;
try {
DateRange range = parseDateRange(dateRange);
@@ -1455,19 +1476,9 @@ public class TimelineReaderWebServices {
long endTime = Time.monotonicNow();
if (entities == null) {
entities = Collections.emptySet();
- } else if (isDisplayEntityPerUserFilterEnabled(config)) {
- Set<TimelineEntity> userEntities = new LinkedHashSet<>();
- userEntities.addAll(entities);
- for (TimelineEntity entity : userEntities) {
- if (entity.getInfo() != null) {
- String userId =
- (String) entity.getInfo().get(FlowActivityEntity.USER_INFO_KEY);
- if (!validateAuthUserWithEntityUser(timelineReaderManager, callerUGI,
- userId)) {
- entities.remove(entity);
- }
- }
- }
+ } else {
+ checkAccess(timelineReaderManager, callerUGI, entities,
+ FlowActivityEntity.USER_INFO_KEY, true);
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@@ -1552,6 +1563,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForAppEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1722,6 +1734,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForAppEntity(entity, callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1852,6 +1865,8 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
+ // TODO to be removed or modified once ACL story is played
+ checkAccess(timelineReaderManager, callerUGI, context.getUserId());
context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
@@ -3343,6 +3358,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForSubAppEntities(entities,callerUGI);
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit");
@@ -3410,6 +3426,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
+ checkAccessForSubAppEntities(entities,callerUGI);
} catch (Exception e) {
handleException(e, url, startTime, "");
}
@@ -3422,7 +3439,7 @@ public class TimelineReaderWebServices {
return entities;
}
- private boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
+ static boolean isDisplayEntityPerUserFilterEnabled(Configuration config) {
return !config
.getBoolean(YarnConfiguration.TIMELINE_SERVICE_READ_AUTH_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED)
@@ -3430,8 +3447,76 @@ public class TimelineReaderWebServices {
.getBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, false);
}
+ // TODO to be removed or modified once ACL story is played
+ private void checkAccessForSubAppEntities(Set<TimelineEntity> entities,
+ UserGroupInformation callerUGI) throws Exception {
+ if (entities != null && entities.size() > 0
+ && isDisplayEntityPerUserFilterEnabled(
+ getTimelineReaderManager().getConfig())) {
+ TimelineReaderContext timelineReaderContext = null;
+ TimelineEntity entity = entities.iterator().next();
+ String fromId =
+ (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+ timelineReaderContext =
+ TimelineFromIdConverter.SUB_APPLICATION_ENTITY_FROMID
+ .decodeUID(fromId);
+ checkAccess(getTimelineReaderManager(), callerUGI,
+ timelineReaderContext.getDoAsUser());
+ }
+ }
+
+ // TODO to be removed or modified once ACL story is played
+ private void checkAccessForAppEntity(TimelineEntity entity,
+ UserGroupInformation callerUGI) throws Exception {
+ if (entity != null && isDisplayEntityPerUserFilterEnabled(
+ getTimelineReaderManager().getConfig())) {
+ String fromId =
+ (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+ TimelineReaderContext timelineReaderContext =
+ TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(fromId);
+ checkAccess(getTimelineReaderManager(), callerUGI,
+ timelineReaderContext.getUserId());
+ }
+ }
+
+ // TODO to be removed or modified once ACL story is played
+ private void checkAccessForGenericEntity(TimelineEntity entity,
+ UserGroupInformation callerUGI) throws Exception {
+ if (entity != null && isDisplayEntityPerUserFilterEnabled(
+ getTimelineReaderManager().getConfig())) {
+ String fromId =
+ (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+ TimelineReaderContext timelineReaderContext =
+ TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(fromId);
+ checkAccess(getTimelineReaderManager(), callerUGI,
+ timelineReaderContext.getUserId());
+ }
+ }
+
+ // TODO to be removed or modified once ACL story is played
+ private void checkAccessForGenericEntities(Set<TimelineEntity> entities,
+ UserGroupInformation callerUGI, String entityType) throws Exception {
+ if (entities != null && entities.size() > 0
+ && isDisplayEntityPerUserFilterEnabled(
+ getTimelineReaderManager().getConfig())) {
+ TimelineReaderContext timelineReaderContext = null;
+ TimelineEntity entity = entities.iterator().next();
+ String uid =
+ (String) entity.getInfo().get(TimelineReaderUtils.FROMID_KEY);
+ if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+ timelineReaderContext =
+ TimelineFromIdConverter.APPLICATION_FROMID.decodeUID(uid);
+ } else {
+ timelineReaderContext =
+ TimelineFromIdConverter.GENERIC_ENTITY_FROMID.decodeUID(uid);
+ }
+ checkAccess(getTimelineReaderManager(), callerUGI,
+ timelineReaderContext.getUserId());
+ }
+ }
+
// TODO to be removed/modified once ACL story has played
- private boolean validateAuthUserWithEntityUser(
+ static boolean validateAuthUserWithEntityUser(
TimelineReaderManager readerManager, UserGroupInformation ugi,
String entityUser) {
String authUser = TimelineReaderWebServicesUtils.getUserName(ugi);
@@ -3442,4 +3527,41 @@ public class TimelineReaderWebServices {
}
return (readerManager.checkAccess(ugi) || authUser.equals(requestedUser));
}
+
+ // TODO to be removed/modified once ACL story has played
+ static boolean checkAccess(TimelineReaderManager readerManager,
+ UserGroupInformation ugi, String entityUser) {
+ if (isDisplayEntityPerUserFilterEnabled(readerManager.getConfig())) {
+ if (!validateAuthUserWithEntityUser(readerManager, ugi, entityUser)) {
+ String userName = ugi.getShortUserName();
+ String msg = "User " + userName
+ + " is not allowed to read TimelineService V2 data.";
+ throw new ForbiddenException(msg);
+ }
+ }
+ return true;
+ }
+
+ // TODO to be removed or modified once ACL story is played
+ static void checkAccess(TimelineReaderManager readerManager,
+ UserGroupInformation callerUGI, Set<TimelineEntity> entities,
+ String entityUserKey, boolean verifyForAllEntity) {
+ if (entities.size() > 0 && isDisplayEntityPerUserFilterEnabled(
+ readerManager.getConfig())) {
+ Set<TimelineEntity> userEntities = new LinkedHashSet<>();
+ userEntities.addAll(entities);
+ for (TimelineEntity entity : userEntities) {
+ if (entity.getInfo() != null) {
+ String userId = (String) entity.getInfo().get(entityUserKey);
+ if (!validateAuthUserWithEntityUser(readerManager, callerUGI,
+ userId)) {
+ entities.remove(entity);
+ if (!verifyForAllEntity) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/469b29c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
new file mode 100644
index 0000000..4239bf0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesBasicAcl.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class TestTimelineReaderWebServicesBasicAcl {
+
+ private TimelineReaderManager manager;
+ private static String adminUser = "admin";
+ private static UserGroupInformation adminUgi =
+ UserGroupInformation.createRemoteUser(adminUser);
+ private Configuration config;
+
+ @Before public void setUp() throws Exception {
+ config = new YarnConfiguration();
+ }
+
+ @After public void tearDown() throws Exception {
+ if (manager != null) {
+ manager.stop();
+ manager = null;
+ }
+ config = null;
+ }
+
+ @Test public void testTimelineReaderManagerAclsWhenDisabled()
+ throws Exception {
+ config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
+ config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
+ manager = new TimelineReaderManager(null);
+ manager.init(config);
+ manager.start();
+
+ // when acls are disabled, always return true
+ Assert.assertTrue(manager.checkAccess(null));
+
+ // filter is disabled, so should return false
+ Assert.assertFalse(
+ TimelineReaderWebServices.isDisplayEntityPerUserFilterEnabled(config));
+ }
+
+ @Test public void testTimelineReaderManagerAclsWhenEnabled()
+ throws Exception {
+ Configuration config = new YarnConfiguration();
+ config.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ config.setBoolean(YarnConfiguration.FILTER_ENTITY_LIST_BY_USER, true);
+ config.set(YarnConfiguration.YARN_ADMIN_ACL, adminUser);
+ manager = new TimelineReaderManager(null);
+ manager.init(config);
+ manager.start();
+
+ String user1 = "user1";
+ String user2 = "user2";
+ UserGroupInformation user1Ugi =
+ UserGroupInformation.createRemoteUser(user1);
+ UserGroupInformation user2Ugi =
+ UserGroupInformation.createRemoteUser(user2);
+
+ // false because ugi is null
+ Assert.assertFalse(TimelineReaderWebServices
+ .validateAuthUserWithEntityUser(manager, null, user1));
+
+ // incoming ugi is admin asking for entity owner user1
+ Assert.assertTrue(
+ TimelineReaderWebServices.checkAccess(manager, adminUgi, user1));
+
+ // incoming ugi is admin asking for entity owner user1
+ Assert.assertTrue(
+ TimelineReaderWebServices.checkAccess(manager, adminUgi, user2));
+
+ // incoming ugi is non-admin i.e user1Ugi asking for entity owner user2
+ try {
+ TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
+ Assert.fail("user1Ugi is not allowed to view user2");
+ } catch (ForbiddenException e) {
+ // expected
+ }
+
+ // incoming ugi is non-admin i.e user2Ugi asking for entity owner user1
+ try {
+ TimelineReaderWebServices.checkAccess(manager, user1Ugi, user2);
+ Assert.fail("user2Ugi is not allowed to view user1");
+ } catch (ForbiddenException e) {
+ // expected
+ }
+
+ String userKey = "user";
+ // incoming ugi is admin asking for entities
+ Set<TimelineEntity> entities = createEntities(10, userKey);
+ TimelineReaderWebServices
+ .checkAccess(manager, adminUgi, entities, userKey, true);
+ // admin is allowed to view other entities
+ Assert.assertTrue(entities.size() == 10);
+
+ // incoming ugi is user1Ugi asking for entities
+ // only user1 entities are allowed to view
+ entities = createEntities(5, userKey);
+ TimelineReaderWebServices
+ .checkAccess(manager, user1Ugi, entities, userKey, true);
+ Assert.assertTrue(entities.size() == 1);
+ Assert
+ .assertEquals(user1, entities.iterator().next().getInfo().get(userKey));
+
+ // incoming ugi is user2Ugi asking for entities
+ // only user2 entities are allowed to view
+ entities = createEntities(8, userKey);
+ TimelineReaderWebServices
+ .checkAccess(manager, user2Ugi, entities, userKey, true);
+ Assert.assertTrue(entities.size() == 1);
+ Assert
+ .assertEquals(user2, entities.iterator().next().getInfo().get(userKey));
+ }
+
+ Set<TimelineEntity> createEntities(int noOfUsers, String userKey) {
+ Set<TimelineEntity> entities = new LinkedHashSet<>();
+ for (int i = 0; i < noOfUsers; i++) {
+ TimelineEntity e = new TimelineEntity();
+ e.setType("user" + i);
+ e.setId("user" + i);
+ e.getInfo().put(userKey, "user" + i);
+ entities.add(e);
+ }
+ return entities;
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HADOOP-14396. Add builder
interface to FileContext. Contributed by Lei (Eddy) Xu.
Posted by gi...@apache.org.
HADOOP-14396. Add builder interface to FileContext.
Contributed by Lei (Eddy) Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ba4e623
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ba4e623
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ba4e623
Branch: refs/heads/HADOOP-15461
Commit: 1ba4e62304a70d53f1a4f76995b6e1fac3107922
Parents: 440140c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 25 14:38:33 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 25 14:38:57 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/FSDataOutputStreamBuilder.java | 22 +++++++
.../java/org/apache/hadoop/fs/FileContext.java | 66 ++++++++++++++++++++
.../main/java/org/apache/hadoop/fs/Options.java | 3 +
.../fs/FileContextMainOperationsBaseTest.java | 44 ++++++++++++-
4 files changed, 134 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 86c284a..d431293 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -116,6 +116,27 @@ public abstract class FSDataOutputStreamBuilder
protected abstract B getThisBuilder();
/**
+ * Construct from a {@link FileContext}.
+ *
+ * @param fc FileContext
+ * @param p path.
+ * @throws IOException
+ */
+ FSDataOutputStreamBuilder(@Nonnull FileContext fc,
+ @Nonnull Path p) throws IOException {
+ Preconditions.checkNotNull(fc);
+ Preconditions.checkNotNull(p);
+ this.fs = null;
+ this.path = p;
+
+ AbstractFileSystem afs = fc.getFSofPath(p);
+ FsServerDefaults defaults = afs.getServerDefaults(p);
+ bufferSize = defaults.getFileBufferSize();
+ replication = defaults.getReplication();
+ blockSize = defaults.getBlockSize();
+ }
+
+ /**
* Constructor.
*/
protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
@@ -131,6 +152,7 @@ public abstract class FSDataOutputStreamBuilder
}
protected FileSystem getFS() {
+ Preconditions.checkNotNull(fs);
return fs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 6ea69d0..5215c3c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -24,6 +24,7 @@ import java.io.OutputStream;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@@ -35,6 +36,8 @@ import java.util.Stack;
import java.util.TreeSet;
import java.util.Map.Entry;
+import javax.annotation.Nonnull;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -695,6 +698,69 @@ public class FileContext {
}
/**
+ * {@link FSDataOutputStreamBuilder} for {@liink FileContext}.
+ */
+ private static final class FCDataOutputStreamBuilder extends
+ FSDataOutputStreamBuilder<
+ FSDataOutputStream, FCDataOutputStreamBuilder> {
+ private final FileContext fc;
+
+ private FCDataOutputStreamBuilder(
+ @Nonnull FileContext fc, @Nonnull Path p) throws IOException {
+ super(fc, p);
+ this.fc = fc;
+ Preconditions.checkNotNull(fc);
+ }
+
+ @Override
+ protected FCDataOutputStreamBuilder getThisBuilder() {
+ return this;
+ }
+
+ @Override
+ public FSDataOutputStream build() throws IOException {
+ final EnumSet<CreateFlag> flags = getFlags();
+ List<CreateOpts> createOpts = new ArrayList<>(Arrays.asList(
+ CreateOpts.blockSize(getBlockSize()),
+ CreateOpts.bufferSize(getBufferSize()),
+ CreateOpts.repFac(getReplication()),
+ CreateOpts.perms(getPermission())
+ ));
+ if (getChecksumOpt() != null) {
+ createOpts.add(CreateOpts.checksumParam(getChecksumOpt()));
+ }
+ if (getProgress() != null) {
+ createOpts.add(CreateOpts.progress(getProgress()));
+ }
+ if (isRecursive()) {
+ createOpts.add(CreateOpts.createParent());
+ }
+ return fc.create(getPath(), flags,
+ createOpts.toArray(new CreateOpts[0]));
+ }
+ }
+
+ /**
+ * Create a {@link FSDataOutputStreamBuilder} for creating or overwriting
+ * a file on indicated path.
+ *
+ * @param f the file path to create builder for.
+ * @return {@link FSDataOutputStreamBuilder} to build a
+ * {@link FSDataOutputStream}.
+ *
+ * Upon {@link FSDataOutputStreamBuilder#build()} being invoked,
+ * builder parameters will be verified by {@link FileContext} and
+ * {@link AbstractFileSystem#create}. And filesystem states will be modified.
+ *
+ * Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
+ * same exceptions as create(Path, EnumSet, CreateOpts...).
+ */
+ public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
+ throws IOException {
+ return new FCDataOutputStreamBuilder(this, f).create();
+ }
+
+ /**
* Make(create) a directory and all the non-existent parents.
*
* @param dir - the dir to make
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
index 126e754..5e93286 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java
@@ -55,6 +55,9 @@ public final class Options {
ChecksumOpt csumOpt) {
return new ChecksumParam(csumOpt);
}
+ public static Progress progress(Progressable prog) {
+ return new Progress(prog);
+ }
public static Perms perms(FsPermission perm) {
return new Perms(perm);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ba4e623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 35ec4ff..62ecd9f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -810,7 +810,49 @@ public abstract class FileContextMainOperationsBaseTest {
fc.create(p, EnumSet.of(CREATE, APPEND, OVERWRITE));
Assert.fail("Excepted exception not thrown");
}
-
+
+ @Test
+ public void testBuilderCreateNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateNonExistingFile");
+ FSDataOutputStream out = fc.create(p).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).overwrite(true).build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendNonExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendNonExistingFile");
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
+ @Test
+ public void testBuilderCreateAppendExistingFile() throws IOException {
+ Path p = getTestRootPath(fc, "test/testBuilderCreateAppendExistingFile");
+ createFile(p);
+ FSDataOutputStream out = fc.create(p).append().build();
+ writeData(fc, p, out, data, 2 * data.length);
+ }
+
+ @Test
+ public void testBuilderCreateRecursive() throws IOException {
+ Path p = getTestRootPath(fc, "test/parent/no/exist/file1");
+ try (FSDataOutputStream out = fc.create(p).build()) {
+ fail("Should throw FileNotFoundException on non-exist directory");
+ } catch (FileNotFoundException e) {
+ }
+
+ FSDataOutputStream out = fc.create(p).recursive().build();
+ writeData(fc, p, out, data, data.length);
+ }
+
private static void writeData(FileContext fc, Path p, FSDataOutputStream out,
byte[] data, long expectedLen) throws IOException {
out.write(data, 0, data.length);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-8461. Support strict memory
control on individual container with elastic control memory mechanism.
Contributed by Haibo Chen.
Posted by gi...@apache.org.
YARN-8461. Support strict memory control on individual container with elastic control memory mechanism. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62d83ca5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62d83ca5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62d83ca5
Branch: refs/heads/HADOOP-15461
Commit: 62d83ca5360cf803ecf6780caf192462d0092009
Parents: b69ba0f
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Tue Jun 26 15:21:35 2018 -0700
Committer: Miklos Szegedi <mi...@cloudera.com>
Committed: Tue Jun 26 15:21:35 2018 -0700
----------------------------------------------------------------------
.../CGroupsMemoryResourceHandlerImpl.java | 24 +++++
.../linux/resources/MemoryResourceHandler.java | 10 ++
.../monitor/ContainersMonitorImpl.java | 108 +++++++++++--------
.../TestCGroupsMemoryResourceHandlerImpl.java | 43 ++++++++
4 files changed, 142 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index a57adb1..053b796 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -34,6 +34,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import java.util.Optional;
+
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
/**
* Handler class to handle the memory controller. YARN already ships a
@@ -172,4 +175,25 @@ public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
return null;
}
+ @Override
+ public Optional<Boolean> isUnderOOM(ContainerId containerId) {
+ try {
+ String status = cGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ containerId.toString(),
+ CGROUP_PARAM_MEMORY_OOM_CONTROL);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("cgroups OOM status for " + containerId + ": " + status);
+ }
+ if (status.contains(CGroupsHandler.UNDER_OOM)) {
+ LOG.warn("Container " + containerId + " under OOM based on cgroups.");
+ return Optional.of(true);
+ } else {
+ return Optional.of(false);
+ }
+ } catch (ResourceHandlerException e) {
+ LOG.warn("Could not read cgroups" + containerId, e);
+ }
+ return Optional.empty();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
index 013a49f..1729fc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/MemoryResourceHandler.java
@@ -20,8 +20,18 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+import java.util.Optional;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public interface MemoryResourceHandler extends ResourceHandler {
+ /**
+ * check whether a container is under OOM.
+ * @param containerId the id of the container
+ * @return empty if the status is unknown, true is the container is under oom,
+ * false otherwise
+ */
+ Optional<Boolean> isUnderOOM(ContainerId containerId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index bd68dfe..d83fe39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupElasticMemoryController;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.MemoryResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import java.util.Arrays;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -697,55 +699,75 @@ public class ContainersMonitorImpl extends AbstractService implements
ProcessTreeInfo ptInfo,
long currentVmemUsage,
long currentPmemUsage) {
- if (elasticMemoryEnforcement || strictMemoryEnforcement) {
- // We enforce the overall memory usage instead of individual containers
- return;
- }
- boolean isMemoryOverLimit = false;
- long vmemLimit = ptInfo.getVmemLimit();
- long pmemLimit = ptInfo.getPmemLimit();
- // as processes begin with an age 1, we want to see if there
- // are processes more than 1 iteration old.
- long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
- long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
+ Optional<Boolean> isMemoryOverLimit = Optional.empty();
String msg = "";
int containerExitStatus = ContainerExitStatus.INVALID;
- if (isVmemCheckEnabled()
- && isProcessTreeOverLimit(containerId.toString(),
- currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
- // The current usage (age=0) is always higher than the aged usage. We
- // do not show the aged size in the message, base the delta on the
- // current usage
- long delta = currentVmemUsage - vmemLimit;
- // Container (the root process) is still alive and overflowing
- // memory.
- // Dump the process-tree and then clean it up.
- msg = formatErrorMessage("virtual",
- formatUsageString(currentVmemUsage, vmemLimit,
+
+ if (strictMemoryEnforcement && elasticMemoryEnforcement) {
+ // Both elastic memory control and strict memory control are enabled
+ // through cgroups. A container will be frozen by the elastic memory
+ // control mechanism if it exceeds its request, so we check for this
+ // here and kill it. Otherwise, the container will not be killed if
+ // the node never exceeds its limit and the procfs-based
+ // memory accounting is different from the cgroup-based accounting.
+
+ MemoryResourceHandler handler =
+ ResourceHandlerModule.getMemoryResourceHandler();
+ if (handler != null) {
+ isMemoryOverLimit = handler.isUnderOOM(containerId);
+ containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+ msg = containerId + " is under oom because it exceeded its" +
+ " physical memory limit";
+ }
+ } else if (strictMemoryEnforcement || elasticMemoryEnforcement) {
+ // if cgroup-based memory control is enabled
+ isMemoryOverLimit = Optional.of(false);
+ }
+
+ if (!isMemoryOverLimit.isPresent()) {
+ long vmemLimit = ptInfo.getVmemLimit();
+ long pmemLimit = ptInfo.getPmemLimit();
+ // as processes begin with an age 1, we want to see if there
+ // are processes more than 1 iteration old.
+ long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
+ long curRssMemUsageOfAgedProcesses = pTree.getRssMemorySize(1);
+ if (isVmemCheckEnabled()
+ && isProcessTreeOverLimit(containerId.toString(),
+ currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) {
+ // The current usage (age=0) is always higher than the aged usage. We
+ // do not show the aged size in the message, base the delta on the
+ // current usage
+ long delta = currentVmemUsage - vmemLimit;
+ // Container (the root process) is still alive and overflowing
+ // memory.
+ // Dump the process-tree and then clean it up.
+ msg = formatErrorMessage("virtual",
+ formatUsageString(currentVmemUsage, vmemLimit,
currentPmemUsage, pmemLimit),
- pId, containerId, pTree, delta);
- isMemoryOverLimit = true;
- containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM;
- } else if (isPmemCheckEnabled()
- && isProcessTreeOverLimit(containerId.toString(),
- currentPmemUsage, curRssMemUsageOfAgedProcesses,
- pmemLimit)) {
- // The current usage (age=0) is always higher than the aged usage. We
- // do not show the aged size in the message, base the delta on the
- // current usage
- long delta = currentPmemUsage - pmemLimit;
- // Container (the root process) is still alive and overflowing
- // memory.
- // Dump the process-tree and then clean it up.
- msg = formatErrorMessage("physical",
- formatUsageString(currentVmemUsage, vmemLimit,
+ pId, containerId, pTree, delta);
+ isMemoryOverLimit = Optional.of(true);
+ containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_VMEM;
+ } else if (isPmemCheckEnabled()
+ && isProcessTreeOverLimit(containerId.toString(),
+ currentPmemUsage, curRssMemUsageOfAgedProcesses,
+ pmemLimit)) {
+ // The current usage (age=0) is always higher than the aged usage. We
+ // do not show the aged size in the message, base the delta on the
+ // current usage
+ long delta = currentPmemUsage - pmemLimit;
+ // Container (the root process) is still alive and overflowing
+ // memory.
+ // Dump the process-tree and then clean it up.
+ msg = formatErrorMessage("physical",
+ formatUsageString(currentVmemUsage, vmemLimit,
currentPmemUsage, pmemLimit),
- pId, containerId, pTree, delta);
- isMemoryOverLimit = true;
- containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+ pId, containerId, pTree, delta);
+ isMemoryOverLimit = Optional.of(true);
+ containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
+ }
}
- if (isMemoryOverLimit) {
+ if (isMemoryOverLimit.isPresent() && isMemoryOverLimit.get()) {
// Virtual or physical memory over limit. Fail the container and
// remove
// the corresponding process tree
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62d83ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
index 5c7e233..4d3e7e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsMemoryResourceHandlerImpl.java
@@ -31,7 +31,9 @@ import org.junit.Test;
import org.junit.Assert;
import java.util.List;
+import java.util.Optional;
+import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler.CGROUP_PARAM_MEMORY_OOM_CONTROL;
import static org.mockito.Mockito.*;
/**
@@ -242,4 +244,45 @@ public class TestCGroupsMemoryResourceHandlerImpl {
.updateCGroupParam(CGroupsHandler.CGroupController.MEMORY, id,
CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES, "1024M");
}
+
+ @Test
+ public void testContainerUnderOom() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+
+ cGroupsMemoryResourceHandler.bootstrap(conf);
+
+ ContainerId containerId = mock(ContainerId.class);
+ when(containerId.toString()).thenReturn("container_01_01");
+
+ when(mockCGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ containerId.toString(),
+ CGROUP_PARAM_MEMORY_OOM_CONTROL)).thenReturn(CGroupsHandler.UNDER_OOM);
+ Optional<Boolean> outOfOom =
+ cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+ Assert.assertTrue("The container should be reported to run under oom",
+ outOfOom.isPresent() && outOfOom.get().equals(true));
+
+ when(mockCGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ containerId.toString(),
+ CGROUP_PARAM_MEMORY_OOM_CONTROL)).thenReturn("");
+ outOfOom = cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+ Assert.assertTrue(
+ "The container should not be reported to run under oom",
+ outOfOom.isPresent() && outOfOom.get().equals(false));
+
+ when(mockCGroupsHandler.getCGroupParam(
+ CGroupsHandler.CGroupController.MEMORY,
+ containerId.toString(),
+ CGROUP_PARAM_MEMORY_OOM_CONTROL)).
+ thenThrow(new ResourceHandlerException());
+ outOfOom = cGroupsMemoryResourceHandler.isUnderOOM(containerId);
+ Assert.assertFalse(
+ "No report of the oom status should be available.",
+ outOfOom.isPresent());
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-15548: Randomize local
dirs. Contributed by Jim Brennan.
Posted by gi...@apache.org.
HADOOP-15548: Randomize local dirs. Contributed by Jim Brennan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d36f6b9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d36f6b9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d36f6b9e
Branch: refs/heads/HADOOP-15461
Commit: d36f6b9e93e4c30d24d0e837cb00bd24ffa8f274
Parents: 1004701
Author: Eric E Payne <er...@oath.com>
Authored: Fri Jun 29 18:18:32 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Fri Jun 29 18:18:32 2018 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/fs/LocalDirAllocator.java | 7 ++-
.../apache/hadoop/fs/TestLocalDirAllocator.java | 59 ++++++++++++++++++++
2 files changed, 65 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36f6b9e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
index c1e9d21..1c216f4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
@@ -418,7 +418,12 @@ public class LocalDirAllocator {
}
}
} else {
- int dirNum = ctx.getAndIncrDirNumLastAccessed();
+ // Start linear search with random increment if possible
+ int randomInc = 1;
+ if (numDirs > 2) {
+ randomInc += dirIndexRandomizer.nextInt(numDirs - 1);
+ }
+ int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc);
while (numDirsSearched < numDirs) {
long capacity = ctx.dirDF[dirNum].getAvailable();
if (capacity > size) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d36f6b9e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
index 825efe0..acda898 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
@@ -265,6 +265,65 @@ public class TestLocalDirAllocator {
}
}
+ /**
+ * Five buffer dirs, on read-write disk.
+ *
+ * Try to create a whole bunch of files.
+ * Verify that each successive creation uses a different disk
+ * than the previous one (for sized requests).
+ *
+ * Would ideally check statistical properties of distribution, but
+ * we don't have the nerve to risk false-positives here.
+ *
+ * @throws Exception
+ */
+ @Test (timeout = 30000)
+ public void testCreateManyFilesRandom() throws Exception {
+ assumeNotWindows();
+ final int numDirs = 5;
+ final int numTries = 100;
+ String[] dirs = new String[numDirs];
+ for (int d = 0; d < numDirs; ++d) {
+ dirs[d] = buildBufferDir(ROOT, d);
+ }
+ boolean next_dir_not_selected_at_least_once = false;
+ try {
+ conf.set(CONTEXT, dirs[0] + "," + dirs[1] + "," + dirs[2] + ","
+ + dirs[3] + "," + dirs[4]);
+ Path[] paths = new Path[5];
+ for (int d = 0; d < numDirs; ++d) {
+ paths[d] = new Path(dirs[d]);
+ assertTrue(localFs.mkdirs(paths[d]));
+ }
+
+ int inDir=0;
+ int prevDir = -1;
+ int[] counts = new int[5];
+ for(int i = 0; i < numTries; ++i) {
+ File result = createTempFile(SMALL_FILE_SIZE);
+ for (int d = 0; d < numDirs; ++d) {
+ if (result.getPath().startsWith(paths[d].toUri().getPath())) {
+ inDir = d;
+ break;
+ }
+ }
+ // Verify we always select a different dir
+ assertNotEquals(prevDir, inDir);
+ // Verify we are not always selecting the next dir - that was the old
+ // algorithm.
+ if ((prevDir != -1) && (inDir != ((prevDir + 1) % numDirs))) {
+ next_dir_not_selected_at_least_once = true;
+ }
+ prevDir = inDir;
+ counts[inDir]++;
+ result.delete();
+ }
+ } finally {
+ rmBufferDirs();
+ }
+ assertTrue(next_dir_not_selected_at_least_once);
+ }
+
/** Two buffer dirs. The first dir does not exist & is on a read-only disk;
* The second dir exists & is RW
* getLocalPathForWrite with checkAccess set to false should create a parent
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by gi...@apache.org.
HDDS-177. Create a releasable ozonefs artifact
Contributed by Marton, Elek.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e16e5b30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e16e5b30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e16e5b30
Branch: refs/heads/HADOOP-15461
Commit: e16e5b307d6c4404db0698b9d128e5bf4aa16a8a
Parents: ca14fec
Author: Anu Engineer <ae...@apache.org>
Authored: Sun Jun 24 01:05:04 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Sun Jun 24 01:05:04 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 4 +-
hadoop-dist/pom.xml | 5 +
.../test/acceptance/ozonefs/docker-compose.yaml | 71 ++
.../src/test/acceptance/ozonefs/docker-config | 39 ++
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 ++
hadoop-ozone/ozonefs/pom.xml | 211 ++++++
.../org/apache/hadoop/fs/ozone/Constants.java | 42 ++
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 ++
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 +++
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 +++++++++++++++++++
.../apache/hadoop/fs/ozone/package-info.java | 30 +
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 +++++
.../fs/ozone/TestOzoneFileInterfaces.java | 231 +++++++
.../contract/ITestOzoneContractCreate.java | 48 ++
.../contract/ITestOzoneContractDelete.java | 48 ++
.../contract/ITestOzoneContractDistCp.java | 50 ++
.../ITestOzoneContractGetFileStatus.java | 61 ++
.../ozone/contract/ITestOzoneContractMkdir.java | 48 ++
.../ozone/contract/ITestOzoneContractOpen.java | 47 ++
.../contract/ITestOzoneContractRename.java | 49 ++
.../contract/ITestOzoneContractRootDir.java | 51 ++
.../ozone/contract/ITestOzoneContractSeek.java | 47 ++
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ++++
.../src/test/resources/contract/ozone.xml | 113 +++
.../ozonefs/src/test/resources/log4j.properties | 23 +
hadoop-ozone/pom.xml | 1 +
hadoop-project/pom.xml | 6 +-
hadoop-tools/hadoop-ozone/pom.xml | 174 -----
.../org/apache/hadoop/fs/ozone/Constants.java | 42 --
.../java/org/apache/hadoop/fs/ozone/OzFs.java | 44 --
.../hadoop/fs/ozone/OzoneFSInputStream.java | 79 ---
.../hadoop/fs/ozone/OzoneFSOutputStream.java | 59 --
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 689 -------------------
.../apache/hadoop/fs/ozone/package-info.java | 30 -
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 157 -----
.../fs/ozone/TestOzoneFileInterfaces.java | 231 -------
.../contract/ITestOzoneContractCreate.java | 48 --
.../contract/ITestOzoneContractDelete.java | 48 --
.../contract/ITestOzoneContractDistCp.java | 50 --
.../ITestOzoneContractGetFileStatus.java | 61 --
.../ozone/contract/ITestOzoneContractMkdir.java | 48 --
.../ozone/contract/ITestOzoneContractOpen.java | 47 --
.../contract/ITestOzoneContractRename.java | 49 --
.../contract/ITestOzoneContractRootDir.java | 51 --
.../ozone/contract/ITestOzoneContractSeek.java | 47 --
.../hadoop/fs/ozone/contract/OzoneContract.java | 123 ----
.../src/test/resources/contract/ozone.xml | 113 ---
.../src/test/resources/log4j.properties | 23 -
hadoop-tools/hadoop-tools-dist/pom.xml | 15 -
hadoop-tools/pom.xml | 11 -
51 files changed, 2413 insertions(+), 2241 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index ad8abe2..be330d5 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -145,6 +145,8 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+mkdir -p "./share/hadoop/ozonefs"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
@@ -153,5 +155,5 @@ cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdd
mkdir -p ./share/hadoop/mapreduce
mkdir -p ./share/hadoop/yarn
echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
+echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone"
echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index dfbf818..5de6759 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -265,6 +265,11 @@
<artifactId>hadoop-ozone-docs</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
new file mode 100644
index 0000000..3323557
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ namenode:
+ image: apache/hadoop-runner
+ hostname: namenode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9870
+ environment:
+ ENSURE_NAMENODE_DIR: /data/namenode
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","namenode"]
+ datanode:
+ image: apache/hadoop-runner
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ksm:
+ image: apache/hadoop-runner
+ hostname: ksm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874
+ environment:
+ ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","ksm"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ hadooplast:
+ image: flokkr/hadoop:3.1.0
+ volumes:
+ - ${OZONEDIR}:/opt/ozone
+ env_file:
+ - ./docker-config
+ environment:
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+ command: ["watch","-n","100000","ls"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
new file mode 100644
index 0000000..dec863e
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.category.org.apache.hadoop.util.NativeCodeLoader=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
new file mode 100644
index 0000000..9e8a5d2
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Ozonefs test
+Library OperatingSystem
+Suite Setup Startup Ozone cluster with size 5
+Suite Teardown Teardown Ozone cluster
+Resource ../commonlib.robot
+
+*** Variables ***
+${COMPOSEFILE} ${CURDIR}/docker-compose.yaml
+${PROJECTDIR} ${CURDIR}/../../../../../..
+
+
+*** Test Cases ***
+Create volume and bucket
+ Execute on datanode ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
+ Execute on datanode ozone oz -createBucket http://ksm/fstest/bucket1
+
+Check volume from ozonefs
+ ${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/
+
+Create directory from ozonefs
+ Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
+ ${result} = Execute on ksm ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
new file mode 100644
index 0000000..c3de4d1
--- /dev/null
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.2.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <name>Apache Hadoop Ozone FileSystem</name>
+ <packaging>jar</packaging>
+ <version>0.2.1-SNAPSHOT</version>
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava:jar</include>
+ <include>org.slf4j:slf4j-api:jar</include>
+ <include>com.google.protobuf:protobuf-java</include>
+ <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+ <include>com.github.stephenc.jcip:jcip-annotations</include>
+ <include>com.google.code.findbugs:jsr305:jar</include>
+ <include>org.apache.hadoop:hadoop-ozone-client</include>
+ <include>org.apache.hadoop:hadoop-hdds-client</include>
+ <include>org.apache.hadoop:hadoop-hdds-common</include>
+ <include>org.fusesource.leveldbjni:leveldbjni-all</include>
+ <include>org.apache.ratis:ratis-server</include>
+ <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+ <include>com.google.auto.value:auto-value-annotations</include>
+ <include>com.squareup:javapoet:jar</include>
+ <include>org.jctools:jctools-core</include>
+ <include>org.apache.ratis:ratis-common</include>
+ <include>org.apache.ratis:ratis-client</include>
+ <include>org.apache.ratis:ratis-netty</include>
+ <include>org.apache.ratis:ratis-grpc</include>
+ <include>org.rocksdb:rocksdbjni</include>
+ <include>org.apache.hadoop:hadoop-ozone-common</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>deplist</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>list</goal>
+ </goals>
+ <configuration>
+ <!-- build a shellprofile -->
+ <outputFile>
+ ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt
+ </outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-scm</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-container-service</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-objectstore-service</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-integration-test</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
new file mode 100644
index 0000000..832a0cb
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+/**
+ * Constants for Ozone FileSystem implementation.
+ */
+public final class Constants {
+
+ public static final String OZONE_DEFAULT_USER = "hdfs";
+
+ public static final String OZONE_USER_DIR = "/user";
+
+ /** Local buffer directory. */
+ public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
+
+ /** Temporary directory. */
+ public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
+
+ /** Page size for Ozone listing operation. */
+ public static final int LISTING_PAGE_SIZE = 1024;
+
+ private Constants() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
new file mode 100644
index 0000000..4163c13
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DelegateToFileSystem;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * ozone implementation of AbstractFileSystem.
+ * This impl delegates to the OzoneFileSystem
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class OzFs extends DelegateToFileSystem {
+
+ public OzFs(URI theUri, Configuration conf)
+ throws IOException, URISyntaxException {
+ super(theUri, new OzoneFileSystem(), conf,
+ OzoneConsts.OZONE_URI_SCHEME, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
new file mode 100644
index 0000000..4c5c0c8
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * The input stream for Ozone file system.
+ *
+ * TODO: Make inputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneFSInputStream extends FSInputStream {
+
+ private final ChunkGroupInputStream inputStream;
+
+ public OzoneFSInputStream(InputStream inputStream) {
+ this.inputStream = (ChunkGroupInputStream)inputStream;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ inputStream.seek(pos);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return inputStream.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return inputStream.available();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
new file mode 100644
index 0000000..faa3628
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
+
+
+/**
+ * The output stream for Ozone file system.
+ *
+ * TODO: Make outputStream generic for both rest and rpc clients
+ * This class is not thread safe.
+ */
+public class OzoneFSOutputStream extends OutputStream {
+
+ private final ChunkGroupOutputStream outputStream;
+
+ public OzoneFSOutputStream(OutputStream outputStream) {
+ this.outputStream = (ChunkGroupOutputStream)outputStream;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ outputStream.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ outputStream.write(b, off, len);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ outputStream.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ outputStream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
new file mode 100644
index 0000000..6906a9d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.OzoneKey;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+
+/**
+ * The Ozone Filesystem implementation.
+ *
+ * This subclass is marked as private as code should not be creating it
+ * directly; use {@link FileSystem#get(Configuration)} and variants to create
+ * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
+ * accessed. Consider those private and unstable.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class OzoneFileSystem extends FileSystem {
+ static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
+
+ /** The Ozone client for connecting to Ozone server. */
+ private OzoneClient ozoneClient;
+ private ObjectStore objectStore;
+ private OzoneVolume volume;
+ private OzoneBucket bucket;
+ private URI uri;
+ private String userName;
+ private Path workingDir;
+ private ReplicationType replicationType;
+ private ReplicationFactor replicationFactor;
+
+ private static final Pattern URL_SCHEMA_PATTERN =
+ Pattern.compile("(.+)\\.([^\\.]+)");
+
+ @Override
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+ setConf(conf);
+ Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
+ assert getScheme().equals(name.getScheme());
+
+ String authority = name.getAuthority();
+
+ Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
+
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Ozone file system url should be "
+ + "in the form o3://bucket.volume");
+ }
+ String bucketStr = matcher.group(1);
+ String volumeStr = matcher.group(2);
+
+ try {
+ uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
+ .setHost(authority).build();
+ LOG.trace("Ozone URI for ozfs initialization is " + uri);
+ this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
+ objectStore = ozoneClient.getObjectStore();
+ this.volume = objectStore.getVolume(volumeStr);
+ this.bucket = volume.getBucket(bucketStr);
+ this.replicationType = ReplicationType.valueOf(
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
+ this.replicationFactor = ReplicationFactor.valueOf(
+ conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
+ try {
+ this.userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ this.userName = OZONE_DEFAULT_USER;
+ }
+ this.workingDir = new Path(OZONE_USER_DIR, this.userName)
+ .makeQualified(this.uri, this.workingDir);
+ } catch (URISyntaxException ue) {
+ final String msg = "Invalid Ozone endpoint " + name;
+ LOG.error(msg, ue);
+ throw new IOException(msg, ue);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ ozoneClient.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public String getScheme() {
+ return OZONE_URI_SCHEME;
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ LOG.trace("open() path:{}", f);
+ final FileStatus fileStatus = getFileStatus(f);
+ final String key = pathToKey(f);
+ if (fileStatus.isDirectory()) {
+ throw new FileNotFoundException("Can't open directory " + f + " to read");
+ }
+
+ return new FSDataInputStream(
+ new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f, FsPermission permission,
+ boolean overwrite, int bufferSize,
+ short replication, long blockSize,
+ Progressable progress) throws IOException {
+ LOG.trace("create() path:{}", f);
+ final String key = pathToKey(f);
+ final FileStatus status;
+ try {
+ status = getFileStatus(f);
+ if (status.isDirectory()) {
+ throw new FileAlreadyExistsException(f + " is a directory");
+ } else {
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(f + " already exists");
+ }
+ LOG.trace("Overwriting file {}", f);
+ deleteObject(key);
+ }
+ } catch (FileNotFoundException ignored) {
+ // check if the parent directory needs to be created
+ Path parent = f.getParent();
+ try {
+ // create all the directories for the parent
+ FileStatus parentStatus = getFileStatus(parent);
+ LOG.trace("parent key:{} status:{}", key, parentStatus);
+ } catch (FileNotFoundException e) {
+ mkdirs(parent);
+ }
+ // This exception needs to ignored as this means that the file currently
+ // does not exists and a new file can thus be created.
+ }
+
+ OzoneOutputStream ozoneOutputStream =
+ bucket.createKey(key, 0, replicationType, replicationFactor);
+ // We pass null to FSDataOutputStream so it won't count writes that
+ // are being buffered to a file
+ return new FSDataOutputStream(
+ new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
+ }
+
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ final Path parent = path.getParent();
+ if (parent != null) {
+ // expect this to raise an exception if there is no parent
+ if (!getFileStatus(parent).isDirectory()) {
+ throw new FileAlreadyExistsException("Not a directory: " + parent);
+ }
+ }
+ return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
+ bufferSize, replication, blockSize, progress);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new UnsupportedOperationException("append() Not implemented by the "
+ + getClass().getSimpleName() + " FileSystem implementation");
+ }
+
+ private class RenameIterator extends OzoneListingIterator {
+ private final String srcKey;
+ private final String dstKey;
+
+ RenameIterator(Path srcPath, Path dstPath)
+ throws IOException {
+ super(srcPath);
+ srcKey = pathToKey(srcPath);
+ dstKey = pathToKey(dstPath);
+ LOG.trace("rename from:{} to:{}", srcKey, dstKey);
+ }
+
+ boolean processKey(String key) throws IOException {
+ String newKeyName = dstKey.concat(key.substring(srcKey.length()));
+ bucket.renameKey(key, newKeyName);
+ return true;
+ }
+ }
+
+ /**
+ * Check whether the source and destination path are valid and then perform
+ * rename from source path to destination path.
+ *
+ * The rename operation is performed by renaming the keys with src as prefix.
+ * For such keys the prefix is changed from src to dst.
+ *
+ * @param src source path for rename
+ * @param dst destination path for rename
+ * @return true if rename operation succeeded or
+ * if the src and dst have the same path and are of the same type
+ * @throws IOException on I/O errors or if the src/dst paths are invalid.
+ */
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ if (src.equals(dst)) {
+ return true;
+ }
+
+ LOG.trace("rename() from:{} to:{}", src, dst);
+ if (src.isRoot()) {
+ // Cannot rename root of file system
+ LOG.trace("Cannot rename the root of a filesystem");
+ return false;
+ }
+
+ // Cannot rename a directory to its own subdirectory
+ Path dstParent = dst.getParent();
+ while (dstParent != null && !src.equals(dstParent)) {
+ dstParent = dstParent.getParent();
+ }
+ Preconditions.checkArgument(dstParent == null,
+ "Cannot rename a directory to its own subdirectory");
+ // Check if the source exists
+ FileStatus srcStatus;
+ try {
+ srcStatus = getFileStatus(src);
+ } catch (FileNotFoundException fnfe) {
+ // source doesn't exist, return
+ return false;
+ }
+
+ // Check if the destination exists
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst parent dir exists or not
+ // if the parent exists, the source can still be renamed to dst path
+ dstStatus = getFileStatus(dst.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", src, dst,
+ dst.getParent()));
+ }
+ } else {
+ // if dst exists and source and destination are same,
+ // check both the src and dst are of same type
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory, rename source as subpath of it.
+ // for example rename /source to /dst will lead to /dst/source
+ dst = new Path(dst, src.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dst);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ src, dst));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", src, dst));
+ }
+ }
+
+ if (srcStatus.isDirectory()) {
+ if (dst.toString().startsWith(src.toString())) {
+ LOG.trace("Cannot rename a directory to a subdirectory of self");
+ return false;
+ }
+ }
+ RenameIterator iterator = new RenameIterator(src, dst);
+ return iterator.iterate();
+ }
+
+ private class DeleteIterator extends OzoneListingIterator {
+ private boolean recursive;
+ DeleteIterator(Path f, boolean recursive)
+ throws IOException {
+ super(f);
+ this.recursive = recursive;
+ if (getStatus().isDirectory()
+ && !this.recursive
+ && listStatus(f).length != 0) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ }
+ }
+
+ boolean processKey(String key) throws IOException {
+ if (key.equals("")) {
+ LOG.trace("Skipping deleting root directory");
+ return true;
+ } else {
+ LOG.trace("deleting key:" + key);
+ boolean succeed = deleteObject(key);
+ // if recursive delete is requested ignore the return value of
+ // deleteObject and issue deletes for other keys.
+ return recursive || succeed;
+ }
+ }
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ LOG.trace("delete() path:{} recursive:{}", f, recursive);
+ try {
+ DeleteIterator iterator = new DeleteIterator(f, recursive);
+ return iterator.iterate();
+ } catch (FileNotFoundException e) {
+ LOG.debug("Couldn't delete {} - does not exist", f);
+ return false;
+ }
+ }
+
+ private class ListStatusIterator extends OzoneListingIterator {
+ private List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
+ private Path f;
+
+ ListStatusIterator(Path f) throws IOException {
+ super(f);
+ this.f = f;
+ }
+
+ boolean processKey(String key) throws IOException {
+ Path keyPath = new Path(OZONE_URI_DELIMITER + key);
+ if (key.equals(getPathKey())) {
+ if (pathIsDirectory()) {
+ return true;
+ } else {
+ statuses.add(getFileStatus(keyPath));
+ return true;
+ }
+ }
+ // left with only subkeys now
+ if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
+ // skip keys which are for subdirectories of the directory
+ statuses.add(getFileStatus(keyPath));
+ }
+ return true;
+ }
+
+ FileStatus[] getStatuses() {
+ return statuses.toArray(new FileStatus[statuses.size()]);
+ }
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f) throws IOException {
+ LOG.trace("listStatus() path:{}", f);
+ ListStatusIterator iterator = new ListStatusIterator(f);
+ iterator.iterate();
+ return iterator.getStatuses();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ workingDir = newDir;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ /**
+ * Check whether the path is valid and then create directories.
+ * Directory is represented using a key with no value.
+ * All the non-existent parent directories are also created.
+ *
+ * @param path directory path to be created
+ * @return true if directory exists or created successfully.
+ * @throws IOException
+ */
+ private boolean mkdir(Path path) throws IOException {
+ Path fPart = path;
+ Path prevfPart = null;
+ do {
+ LOG.trace("validating path:{}", fPart);
+ try {
+ FileStatus fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ // If path exists and a directory, exit
+ break;
+ } else {
+ // Found a file here, rollback and delete newly created directories
+ LOG.trace("Found a file with same name as directory, path:{}", fPart);
+ if (prevfPart != null) {
+ delete(prevfPart, true);
+ }
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s', it is a file.", fPart));
+ }
+ } catch (FileNotFoundException fnfe) {
+ LOG.trace("creating directory for fpart:{}", fPart);
+ String key = pathToKey(fPart);
+ String dirKey = addTrailingSlashIfNeeded(key);
+ if (!createDirectory(dirKey)) {
+ // Directory creation failed here,
+ // rollback and delete newly created directories
+ LOG.trace("Directory creation failed, path:{}", fPart);
+ if (prevfPart != null) {
+ delete(prevfPart, true);
+ }
+ return false;
+ }
+ }
+ prevfPart = fPart;
+ fPart = fPart.getParent();
+ } while (fPart != null);
+ return true;
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ LOG.trace("mkdir() path:{} ", f);
+ String key = pathToKey(f);
+ if (StringUtils.isEmpty(key)) {
+ return false;
+ }
+ return mkdir(f);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ LOG.trace("getFileStatus() path:{}", f);
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+
+ if (key.length() == 0) {
+ return new FileStatus(0, true, 1, 0,
+ bucket.getCreationTime(), qualifiedPath);
+ }
+
+ // consider this a file and get key status
+ OzoneKey meta = getKeyInfo(key);
+ if (meta == null) {
+ key = addTrailingSlashIfNeeded(key);
+ meta = getKeyInfo(key);
+ }
+
+ if (meta == null) {
+ LOG.trace("File:{} not found", f);
+ throw new FileNotFoundException(f + ": No such file or directory!");
+ } else if (isDirectory(meta)) {
+ return new FileStatus(0, true, 1, 0,
+ meta.getModificationTime(), qualifiedPath);
+ } else {
+ //TODO: Fetch replication count from ratis config
+ return new FileStatus(meta.getDataSize(), false, 1,
+ getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
+ }
+ }
+
+ /**
+ * Helper method to fetch the key metadata info.
+ * @param key key whose metadata information needs to be fetched
+ * @return metadata info of the key
+ */
+ private OzoneKey getKeyInfo(String key) {
+ try {
+ return bucket.getKey(key);
+ } catch (IOException e) {
+ LOG.trace("Key:{} does not exists", key);
+ return null;
+ }
+ }
+
+ /**
+ * Helper method to check if an Ozone key is representing a directory.
+ * @param key key to be checked as a directory
+ * @return true if key is a directory, false otherwise
+ */
+ private boolean isDirectory(OzoneKey key) {
+ LOG.trace("key name:{} size:{}", key.getName(),
+ key.getDataSize());
+ return key.getName().endsWith(OZONE_URI_DELIMITER)
+ && (key.getDataSize() == 0);
+ }
+
+ /**
+ * Helper method to create an directory specified by key name in bucket.
+ * @param keyName key name to be created as directory
+ * @return true if the key is created, false otherwise
+ */
+ private boolean createDirectory(String keyName) {
+ try {
+ LOG.trace("creating dir for key:{}", keyName);
+ bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("create key failed for key:{}", keyName, ioe);
+ return false;
+ }
+ }
+
+ /**
+ * Helper method to delete an object specified by key name in bucket.
+ * @param keyName key name to be deleted
+ * @return true if the key is deleted, false otherwise
+ */
+ private boolean deleteObject(String keyName) {
+ LOG.trace("issuing delete for key" + keyName);
+ try {
+ bucket.deleteKey(keyName);
+ return true;
+ } catch (IOException ioe) {
+ LOG.error("delete key failed " + ioe.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an Ozone key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ public String pathToKey(Path path) {
+ Objects.requireNonNull(path, "Path can not be null!");
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+ // removing leading '/' char
+ String key = path.toUri().getPath().substring(1);
+ LOG.trace("path for key:{} is:{}", key, path);
+ return key;
+ }
+
+ /**
+ * Add trailing delimiter to path if it is already not present.
+ *
+ * @param key the ozone Key which needs to be appended
+ * @return delimiter appended key
+ */
+ private String addTrailingSlashIfNeeded(String key) {
+ if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
+ return key + OZONE_URI_DELIMITER;
+ } else {
+ return key;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "OzoneFileSystem{URI=" + uri + ", "
+ + "workingDir=" + workingDir + ", "
+ + "userName=" + userName + ", "
+ + "statistics=" + statistics
+ + "}";
+ }
+
+ private abstract class OzoneListingIterator {
+ private final Path path;
+ private final FileStatus status;
+ private String pathKey;
+ private Iterator<OzoneKey> keyIterator;
+
+ OzoneListingIterator(Path path)
+ throws IOException {
+ this.path = path;
+ this.status = getFileStatus(path);
+ this.pathKey = pathToKey(path);
+ if (status.isDirectory()) {
+ this.pathKey = addTrailingSlashIfNeeded(pathKey);
+ }
+ keyIterator = bucket.listKeys(pathKey);
+ }
+
+ abstract boolean processKey(String key) throws IOException;
+
+ // iterates all the keys in the particular path
+ boolean iterate() throws IOException {
+ LOG.trace("Iterating path {}", path);
+ if (status.isDirectory()) {
+ LOG.trace("Iterating directory:{}", pathKey);
+ while (keyIterator.hasNext()) {
+ OzoneKey key = keyIterator.next();
+ LOG.trace("iterating key:{}", key.getName());
+ if (!processKey(key.getName())) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ LOG.trace("iterating file:{}", path);
+ return processKey(pathKey);
+ }
+ }
+
+ String getPathKey() {
+ return pathKey;
+ }
+
+ boolean pathIsDirectory() {
+ return status.isDirectory();
+ }
+
+ FileStatus getStatus() {
+ return status;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
new file mode 100644
index 0000000..93e82c3
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Ozone Filesystem.
+ *
+ * Except for the exceptions, it should all be hidden as implementation details.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
new file mode 100644
index 0000000..ad21f28
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Test OzoneFSInputStream by reading through multiple interfaces.
+ */
+public class TestOzoneFSInputStream {
+ private static MiniOzoneCluster cluster = null;
+ private static FileSystem fs;
+ private static StorageHandler storageHandler;
+ private static Path filePath = null;
+ private static byte[] data = null;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(10)
+ .build();
+ cluster.waitForClusterToBeReady();
+ storageHandler =
+ new ObjectStoreHandler(conf).getStorageHandler();
+
+ // create a volume and a bucket to be used by OzoneFileSystem
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ storageHandler.createVolume(volumeArgs);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ storageHandler.createBucket(bucketArgs);
+
+ // Fetch the host and port for File System init
+ DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
+ .getDatanodeDetails();
+ int port = datanodeDetails
+ .getPort(DatanodeDetails.Port.Name.REST).getValue();
+ String host = datanodeDetails.getHostName();
+
+ // Set the fs.defaultFS and start the filesystem
+ String uri = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
+ fs = FileSystem.get(conf);
+ int fileLen = 100 * 1024 * 1024;
+ data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
+ filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
+ try (FSDataOutputStream stream = fs.create(filePath)) {
+ stream.write(data);
+ }
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() throws IOException {
+ fs.close();
+ storageHandler.close();
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testO3FSSingleByteRead() throws IOException {
+ FSDataInputStream inputStream = fs.open(filePath);
+ byte[] value = new byte[data.length];
+ int i = 0;
+ while(true) {
+ int val = inputStream.read();
+ if (val == -1) {
+ break;
+ }
+ value[i] = (byte)val;
+ Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
+ i++;
+ }
+ Assert.assertEquals(i, data.length);
+ Assert.assertTrue(Arrays.equals(value, data));
+ inputStream.close();
+ }
+
+ @Test
+ public void testO3FSMultiByteRead() throws IOException {
+ FSDataInputStream inputStream = fs.open(filePath);
+ byte[] value = new byte[data.length];
+ byte[] tmp = new byte[1* 1024 *1024];
+ int i = 0;
+ while(true) {
+ int val = inputStream.read(tmp);
+ if (val == -1) {
+ break;
+ }
+ System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
+ i++;
+ }
+ Assert.assertEquals(i * tmp.length, data.length);
+ Assert.assertTrue(Arrays.equals(value, data));
+ inputStream.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
new file mode 100644
index 0000000..a225702
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
+
+import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test OzoneFileSystem Interfaces.
+ *
+ * This test will test the various interfaces i.e.
+ * create, read, write, getFileStatus
+ */
+@RunWith(Parameterized.class)
+public class TestOzoneFileInterfaces {
+
+ private String rootPath;
+ private String userName;
+
+ /**
+ * Parameter class to set absolute url/defaultFS handling.
+ * <p>
+ * Hadoop file systems could be used in multiple ways: Using the defaultfs
+ * and file path without the schema, or use absolute url-s even with
+ * different defaultFS. This parameter matrix would test both the use cases.
+ */
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {{false, true}, {true, false}});
+ }
+
+ private boolean setDefaultFs;
+
+ private boolean useAbsolutePath;
+
+ private static MiniOzoneCluster cluster = null;
+
+ private static FileSystem fs;
+
+ private static StorageHandler storageHandler;
+
+ public TestOzoneFileInterfaces(boolean setDefaultFs,
+ boolean useAbsolutePath) {
+ this.setDefaultFs = setDefaultFs;
+ this.useAbsolutePath = useAbsolutePath;
+ }
+
+ @Before
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+ storageHandler =
+ new ObjectStoreHandler(conf).getStorageHandler();
+
+ // create a volume and a bucket to be used by OzoneFileSystem
+ userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ storageHandler.createVolume(volumeArgs);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ storageHandler.createBucket(bucketArgs);
+
+ rootPath = String
+ .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
+ volumeName);
+ if (setDefaultFs) {
+ // Set the fs.defaultFS and start the filesystem
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
+ fs = FileSystem.get(conf);
+ } else {
+ fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
+ }
+ }
+
+ @After
+ public void teardown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.closeQuietly(fs);
+ IOUtils.closeQuietly(storageHandler);
+ }
+
+ @Test
+ public void testFileSystemInit() throws IOException {
+ if (setDefaultFs) {
+ assertTrue(
+ "The initialized file system is not OzoneFileSystem but " +
+ fs.getClass(),
+ fs instanceof OzoneFileSystem);
+ assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
+ }
+ }
+
+ @Test
+ public void testOzFsReadWrite() throws IOException {
+ long currentTime = Time.now();
+ int stringLen = 20;
+ String data = RandomStringUtils.randomAlphanumeric(stringLen);
+ String filePath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + filePath);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ stream.writeBytes(data);
+ }
+
+ FileStatus status = fs.getFileStatus(path);
+ // The timestamp of the newly created file should always be greater than
+ // the time when the test was started
+ assertTrue("Modification time has not been recorded: " + status,
+ status.getModificationTime() > currentTime);
+
+ try (FSDataInputStream inputStream = fs.open(path)) {
+ byte[] buffer = new byte[stringLen];
+ inputStream.readFully(0, buffer);
+ String out = new String(buffer, 0, buffer.length);
+ assertEquals(data, out);
+ }
+ }
+
+
+ @Test
+ public void testDirectory() throws IOException {
+ String dirPath = RandomStringUtils.randomAlphanumeric(5);
+ Path path = createPath("/" + dirPath);
+ assertTrue("Makedirs returned with false for the path " + path,
+ fs.mkdirs(path));
+
+ FileStatus status = fs.getFileStatus(path);
+ assertTrue("The created path is not directory.", status.isDirectory());
+
+ assertEquals(0, status.getLen());
+
+ FileStatus[] statusList = fs.listStatus(createPath("/"));
+ assertEquals(1, statusList.length);
+ assertEquals(status, statusList[0]);
+
+ FileStatus statusRoot = fs.getFileStatus(createPath("/"));
+ assertTrue("Root dir (/) is not a directory.", status.isDirectory());
+ assertEquals(0, status.getLen());
+
+
+ }
+
+ @Test
+ public void testPathToKey() throws Exception {
+ OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
+
+ assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
+
+ assertEquals("user/" + getCurrentUser() + "/key1/key2",
+ ozoneFs.pathToKey(new Path("key1/key2")));
+
+ assertEquals("key1/key2",
+ ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
+ }
+
+ private String getCurrentUser() {
+ try {
+ return UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException e) {
+ return OZONE_DEFAULT_USER;
+ }
+ }
+
+ private Path createPath(String relativePath) {
+ if (useAbsolutePath) {
+ return new Path(
+ rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
+ } else {
+ return new Path(relativePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
new file mode 100644
index 0000000..dd54315
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests creating files.
+ */
+public class ITestOzoneContractCreate extends AbstractContractCreateTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
new file mode 100644
index 0000000..f0a3d8d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering deletes.
+ */
+public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
new file mode 100644
index 0000000..134a9ad
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ * Uses the block output stream, buffered to disk. This is the
+ * recommended output mechanism for DistCP due to its scalability.
+ */
+public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected OzoneContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
new file mode 100644
index 0000000..98bbb14
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering getFileStatus.
+ */
+public class ITestOzoneContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ getLog().info("FS details {}", getFileSystem());
+ super.teardown();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ return super.createConfiguration();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
new file mode 100644
index 0000000..bc0de5d
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Test dir operations on Ozone.
+ */
+public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
new file mode 100644
index 0000000..0bc57d4
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests opening files.
+ */
+public class ITestOzoneContractOpen extends AbstractContractOpenTest {
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
new file mode 100644
index 0000000..8ce1d1b
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering rename.
+ */
+public class ITestOzoneContractRename extends AbstractContractRenameTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-13707. [PROVIDED Storage] Fix
failing integration tests in ITestProvidedImplementation. Contributed by
Virajith Jalaparti.
Posted by gi...@apache.org.
HDFS-13707. [PROVIDED Storage] Fix failing integration tests in ITestProvidedImplementation. Contributed by Virajith Jalaparti.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73746c5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73746c5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73746c5d
Branch: refs/heads/HADOOP-15461
Commit: 73746c5da76d5e39df131534a1ec35dfc5d2529b
Parents: e4d7227
Author: Inigo Goiri <in...@apache.org>
Authored: Fri Jun 29 09:56:13 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Fri Jun 29 09:56:13 2018 -0700
----------------------------------------------------------------------
.../common/blockaliasmap/impl/TextFileRegionAliasMap.java | 5 +++--
.../hdfs/server/namenode/ITestProvidedImplementation.java | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73746c5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
index abe92e3..4d65142 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TextFileRegionAliasMap.java
@@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Iterator;
import java.util.Map;
import java.util.Collections;
@@ -359,7 +360,7 @@ public class TextFileRegionAliasMap
}
byte[] nonce = new byte[0];
if (f.length == 6) {
- nonce = f[5].getBytes(Charset.forName("UTF-8"));
+ nonce = Base64.getDecoder().decode(f[5]);
}
return new FileRegion(Long.parseLong(f[0]), new Path(f[1]),
Long.parseLong(f[2]), Long.parseLong(f[3]), Long.parseLong(f[4]),
@@ -451,7 +452,7 @@ public class TextFileRegionAliasMap
out.append(Long.toString(block.getGenerationStamp()));
if (psl.getNonce().length > 0) {
out.append(delim)
- .append(new String(psl.getNonce(), Charset.forName("UTF-8")));
+ .append(Base64.getEncoder().encodeToString(psl.getNonce()));
}
out.append("\n");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73746c5d/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
index 49c9bcf..7d3ab0e 100644
--- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
+++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/ITestProvidedImplementation.java
@@ -132,7 +132,7 @@ public class ITestProvidedImplementation {
nnDirPath.toString());
conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_READ_FILE,
new Path(nnDirPath, fileNameFromBlockPoolID(bpid)).toString());
- conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_DELIMITER, ",");
+ conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_TEXT_DELIMITER, "\t");
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR_PROVIDED,
new File(providedPath.toUri()).toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-194. Remove NodePoolManager
and node pool handling from SCM. Contributed by Elek Marton.
Posted by gi...@apache.org.
HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56a4cdb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56a4cdb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56a4cdb9
Branch: refs/heads/HADOOP-15461
Commit: 56a4cdb9804daea7164155a5b1b4eba44a11b705
Parents: 0d6fe5f
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:28:00 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:28:00 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 --
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 -
.../common/src/main/resources/ozone-default.xml | 47 ---
.../hdds/scm/container/ContainerMapping.java | 10 +-
.../replication/ContainerSupervisor.java | 340 -------------------
.../container/replication/InProgressPool.java | 255 --------------
.../scm/container/replication/PeriodicPool.java | 119 -------
.../scm/container/replication/package-info.java | 23 --
.../hadoop/hdds/scm/node/NodeManager.java | 6 -
.../hadoop/hdds/scm/node/NodePoolManager.java | 71 ----
.../hadoop/hdds/scm/node/SCMNodeManager.java | 23 --
.../hdds/scm/node/SCMNodePoolManager.java | 269 ---------------
.../hdds/scm/container/MockNodeManager.java | 6 -
.../hdds/scm/node/TestSCMNodePoolManager.java | 160 ---------
.../testutils/ReplicationNodeManagerMock.java | 5 -
.../ReplicationNodePoolManagerMock.java | 133 --------
.../hadoop/ozone/scm/TestContainerSQLCli.java | 31 --
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 74 ----
18 files changed, 3 insertions(+), 1596 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 85407e6..df6fbf0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,32 +243,6 @@ public final class ScmConfigKeys {
public static final String
OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
- /**
- * Don't start processing a pool if we have not had a minimum number of
- * seconds from the last processing.
- */
- public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
- "ozone.scm.container.report.processing.interval";
- public static final String
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
- /**
- * This determines the total number of pools to be processed in parallel.
- */
- public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
- "ozone.scm.max.nodepool.processing.threads";
- public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
- /**
- * These 2 settings control the number of threads in executor pool and time
- * outs for thw container reports from all nodes.
- */
- public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
- "ozone.scm.max.container.report.threads";
- public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
- "ozone.scm.container.reports.wait.timeout";
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
- "5m";
public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
"ozone.scm.block.deletion.max.retry";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index c40dc8e..08a5ffd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,7 +91,6 @@ public final class OzoneConsts {
public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String BLOCK_DB = "block.db";
- public static final String NODEPOOL_DB = "nodepool.db";
public static final String OPEN_CONTAINERS_DB = "openContainers.db";
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String KSM_DB_NAME = "ksm.db";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7a91610..25365c8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,25 +572,6 @@
</description>
</property>
<property>
- <name>ozone.scm.container.report.processing.interval</name>
- <value>60s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Time interval for scm to process container reports
- for a node pool. Scm handles node pool reports in a cyclic clock
- manner, it fetches pools periodically with this time interval.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.reports.wait.timeout</name>
- <value>300s</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>Maximum time to wait in seconds for processing all container
- reports from
- a node pool. It determines the timeout for a
- node pool report.
- </description>
- </property>
- <property>
<name>ozone.scm.container.size.gb</name>
<value>5</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -793,17 +774,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.container.report.threads</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Maximum number of threads to process container reports in scm.
- Each container report from a data node is processed by scm in a worker
- thread, fetched from a thread pool. This property is used to control the
- maximum size of the thread pool.
- </description>
- </property>
- <property>
<name>ozone.scm.max.hb.count.to.process</name>
<value>5000</value>
<tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -815,14 +785,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- Number of node pools to process in parallel.
- </description>
- </property>
- <property>
<name>ozone.scm.names</name>
<value/>
<tag>OZONE</tag>
@@ -844,15 +806,6 @@
</description>
</property>
<property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, SCM</tag>
- <description>
- Controls the number of node pools that can be processed in parallel by
- Container Supervisor.
- </description>
- </property>
- <property>
<name>ozone.trace.enabled</name>
<value>false</value>
<tag>OZONE, DEBUG</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index b563e90..9fd30f2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -80,7 +79,6 @@ public class ContainerMapping implements Mapping {
private final PipelineSelector pipelineSelector;
private final ContainerStateManager containerStateManager;
private final LeaseManager<ContainerInfo> containerLeaseManager;
- private final ContainerSupervisor containerSupervisor;
private final float containerCloseThreshold;
private final ContainerCloser closer;
private final long size;
@@ -127,9 +125,7 @@ public class ContainerMapping implements Mapping {
OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
this.containerStateManager =
new ContainerStateManager(conf, this);
- this.containerSupervisor =
- new ContainerSupervisor(conf, nodeManager,
- nodeManager.getNodePoolManager());
+
this.containerCloseThreshold = conf.getFloat(
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -407,8 +403,8 @@ public class ContainerMapping implements Mapping {
throws IOException {
List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
containerInfos = reports.getReportsList();
- containerSupervisor.handleContainerReport(datanodeDetails, reports);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
containerInfos) {
byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
lock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index 5bd0574..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
- .sleepUninterruptibly;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
- public static final Logger LOG =
- LoggerFactory.getLogger(ContainerSupervisor.class);
-
- private final NodePoolManager poolManager;
- private final HashSet<String> poolNames;
- private final PriorityQueue<PeriodicPool> poolQueue;
- private final NodeManager nodeManager;
- private final long containerProcessingLag;
- private final AtomicBoolean runnable;
- private final ExecutorService executorService;
- private final long maxPoolWait;
- private long poolProcessCount;
- private final List<InProgressPool> inProgressPoolList;
- private final AtomicInteger threadFaultCount;
- private final int inProgressPoolMaxCount;
-
- private final ReadWriteLock inProgressPoolListLock;
-
- /**
- * Returns the number of times we have processed pools.
- * @return long
- */
- public long getPoolProcessCount() {
- return poolProcessCount;
- }
-
-
- /**
- * Constructs a class that computes Replication Levels.
- *
- * @param conf - OzoneConfiguration
- * @param nodeManager - Node Manager
- * @param poolManager - Pool Manager
- */
- public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
- NodePoolManager poolManager) {
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(nodeManager);
- this.containerProcessingLag =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
- TimeUnit.SECONDS
- ) * 1000;
- int maxContainerReportThreads =
- conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
- OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
- );
- this.maxPoolWait =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
- OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.inProgressPoolMaxCount = conf.getInt(
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
- this.poolManager = poolManager;
- this.nodeManager = nodeManager;
- this.poolNames = new HashSet<>();
- this.poolQueue = new PriorityQueue<>();
- this.runnable = new AtomicBoolean(true);
- this.threadFaultCount = new AtomicInteger(0);
- this.executorService = newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Container Reports Processing Thread - %d")
- .build(), maxContainerReportThreads);
- this.inProgressPoolList = new LinkedList<>();
- this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
- initPoolProcessThread();
- }
-
- private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
- int maxThreads) {
- return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
- new LinkedBlockingQueue<>(), threadFactory);
- }
-
- /**
- * Returns the number of pools that are under process right now.
- * @return int - Number of pools that are in process.
- */
- public int getInProgressPoolCount() {
- return inProgressPoolList.size();
- }
-
- /**
- * Exits the background thread.
- */
- public void setExit() {
- this.runnable.set(false);
- }
-
- /**
- * Adds or removes pools from names that we need to process.
- *
- * There are two different cases that we need to process.
- * The case where some pools are being added and some times we have to
- * handle cases where pools are removed.
- */
- private void refreshPools() {
- List<String> pools = this.poolManager.getNodePools();
- if (pools != null) {
-
- HashSet<String> removedPools =
- computePoolDifference(this.poolNames, new HashSet<>(pools));
-
- HashSet<String> addedPools =
- computePoolDifference(new HashSet<>(pools), this.poolNames);
- // TODO: Support remove pool API in pool manager so that this code
- // path can be tested. This never happens in the current code base.
- for (String poolName : removedPools) {
- for (PeriodicPool periodicPool : poolQueue) {
- if (periodicPool.getPoolName().compareTo(poolName) == 0) {
- poolQueue.remove(periodicPool);
- }
- }
- }
- // Remove the pool names that we have in the list.
- this.poolNames.removeAll(removedPools);
-
- for (String poolName : addedPools) {
- poolQueue.add(new PeriodicPool(poolName));
- }
-
- // Add to the pool names we are tracking.
- poolNames.addAll(addedPools);
- }
-
- }
-
- /**
- * Handle the case where pools are added.
- *
- * @param newPools - New Pools list
- * @param oldPool - oldPool List.
- */
- private HashSet<String> computePoolDifference(HashSet<String> newPools,
- Set<String> oldPool) {
- Preconditions.checkNotNull(newPools);
- Preconditions.checkNotNull(oldPool);
- HashSet<String> newSet = new HashSet<>(newPools);
- newSet.removeAll(oldPool);
- return newSet;
- }
-
- private void initPoolProcessThread() {
-
- /*
- * Task that runs to check if we need to start a pool processing job.
- * if so we create a pool reconciliation job and find out of all the
- * expected containers are on the nodes.
- */
- Runnable processPools = () -> {
- while (runnable.get()) {
- // Make sure that we don't have any new pools.
- refreshPools();
- while (inProgressPoolList.size() < inProgressPoolMaxCount) {
- PeriodicPool pool = poolQueue.poll();
- if (pool != null) {
- if (pool.getLastProcessedTime() + this.containerProcessingLag >
- Time.monotonicNow()) {
- LOG.debug("Not within the time window for processing: {}",
- pool.getPoolName());
- // we might over sleep here, not a big deal.
- sleepUninterruptibly(this.containerProcessingLag,
- TimeUnit.MILLISECONDS);
- }
- LOG.debug("Adding pool {} to container processing queue",
- pool.getPoolName());
- InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
- pool, this.nodeManager, this.poolManager, this.executorService);
- inProgressPool.startReconciliation();
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.add(inProgressPool);
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- poolProcessCount++;
- } else {
- break;
- }
- }
- sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
- inProgressPoolListLock.readLock().lock();
- try {
- for (InProgressPool inProgressPool : inProgressPoolList) {
- inProgressPool.finalizeReconciliation();
- poolQueue.add(inProgressPool.getPool());
- }
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.clear();
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- }
- };
-
- // We will have only one thread for pool processing.
- Thread poolProcessThread = new Thread(processPools);
- poolProcessThread.setDaemon(true);
- poolProcessThread.setName("Pool replica thread");
- poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
- // Let us just restart this thread after logging a critical error.
- // if this thread is not running we cannot handle commands from SCM.
- LOG.error("Critical Error : Pool replica thread encountered an " +
- "error. Thread: {} Error Count : {}", t.toString(), e,
- threadFaultCount.incrementAndGet());
- poolProcessThread.start();
- // TODO : Add a config to restrict how many times we will restart this
- // thread in a single session.
- });
- poolProcessThread.start();
- }
-
- /**
- * Adds a container report to appropriate inProgress Pool.
- * @param containerReport -- Container report for a specific container from
- * a datanode.
- */
- public void handleContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto containerReport) {
- inProgressPoolListLock.readLock().lock();
- try {
- String poolName = poolManager.getNodePool(datanodeDetails);
- for (InProgressPool ppool : inProgressPoolList) {
- if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
- ppool.handleContainerReport(datanodeDetails, containerReport);
- return;
- }
- }
- // TODO: Decide if we can do anything else with this report.
- LOG.debug("Discarding the container report for pool {}. " +
- "That pool is not currently in the pool reconciliation process." +
- " Container Name: {}", poolName, datanodeDetails);
- } catch (SCMException e) {
- LOG.warn("Skipping processing container report from datanode {}, "
- + "cause: failed to get the corresponding node pool",
- datanodeDetails.toString(), e);
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- }
-
- /**
- * Get in process pool list, used for testing.
- * @return List of InProgressPool
- */
- @VisibleForTesting
- public List<InProgressPool> getInProcessPoolList() {
- return inProgressPoolList;
- }
-
- /**
- * Shutdown the Container Replication Manager.
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- setExit();
- HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 4b54731..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
- public static final Logger LOG =
- LoggerFactory.getLogger(InProgressPool.class);
-
- private final PeriodicPool pool;
- private final NodeManager nodeManager;
- private final NodePoolManager poolManager;
- private final ExecutorService executorService;
- private final Map<Long, Integer> containerCountMap;
- private final Map<UUID, Boolean> processedNodeSet;
- private final long startTime;
- private ProgressStatus status;
- private AtomicInteger nodeCount;
- private AtomicInteger nodeProcessed;
- private AtomicInteger containerProcessedCount;
- private long maxWaitTime;
- /**
- * Constructs an pool that is being processed.
- * @param maxWaitTime - Maximum wait time in milliseconds.
- * @param pool - Pool that we are working against
- * @param nodeManager - Nodemanager
- * @param poolManager - pool manager
- * @param executorService - Shared Executor service.
- */
- InProgressPool(long maxWaitTime, PeriodicPool pool,
- NodeManager nodeManager, NodePoolManager poolManager,
- ExecutorService executorService) {
- Preconditions.checkNotNull(pool);
- Preconditions.checkNotNull(nodeManager);
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(executorService);
- Preconditions.checkArgument(maxWaitTime > 0);
- this.pool = pool;
- this.nodeManager = nodeManager;
- this.poolManager = poolManager;
- this.executorService = executorService;
- this.containerCountMap = new ConcurrentHashMap<>();
- this.processedNodeSet = new ConcurrentHashMap<>();
- this.maxWaitTime = maxWaitTime;
- startTime = Time.monotonicNow();
- }
-
- /**
- * Returns periodic pool.
- *
- * @return PeriodicPool
- */
- public PeriodicPool getPool() {
- return pool;
- }
-
- /**
- * We are done if we have got reports from all nodes or we have
- * done waiting for the specified time.
- *
- * @return true if we are done, false otherwise.
- */
- public boolean isDone() {
- return (nodeCount.get() == nodeProcessed.get()) ||
- (this.startTime + this.maxWaitTime) > Time.monotonicNow();
- }
-
- /**
- * Gets the number of containers processed.
- *
- * @return int
- */
- public int getContainerProcessedCount() {
- return containerProcessedCount.get();
- }
-
- /**
- * Returns the start time in milliseconds.
- *
- * @return - Start Time.
- */
- public long getStartTime() {
- return startTime;
- }
-
- /**
- * Get the number of nodes in this pool.
- *
- * @return - node count
- */
- public int getNodeCount() {
- return nodeCount.get();
- }
-
- /**
- * Get the number of nodes that we have already processed container reports
- * from.
- *
- * @return - Processed count.
- */
- public int getNodeProcessed() {
- return nodeProcessed.get();
- }
-
- /**
- * Returns the current status.
- *
- * @return Status
- */
- public ProgressStatus getStatus() {
- return status;
- }
-
- /**
- * Starts the reconciliation process for all the nodes in the pool.
- */
- public void startReconciliation() {
- List<DatanodeDetails> datanodeDetailsList =
- this.poolManager.getNodes(pool.getPoolName());
- if (datanodeDetailsList.size() == 0) {
- LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
- pool.getPoolName());
- this.status = ProgressStatus.Error;
- return;
- }
-
- nodeProcessed = new AtomicInteger(0);
- containerProcessedCount = new AtomicInteger(0);
- nodeCount = new AtomicInteger(0);
- this.status = ProgressStatus.InProgress;
- this.getPool().setLastProcessedTime(Time.monotonicNow());
- }
-
- /**
- * Queues a container Report for handling. This is done in a worker thread
- * since decoding a container report might be compute intensive . We don't
- * want to block since we have asked for bunch of container reports
- * from a set of datanodes.
- *
- * @param containerReport - ContainerReport
- */
- public void handleContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto containerReport) {
- if (status == ProgressStatus.InProgress) {
- executorService.submit(processContainerReport(datanodeDetails,
- containerReport));
- } else {
- LOG.debug("Cannot handle container report when the pool is in {} status.",
- status);
- }
- }
-
- private Runnable processContainerReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports) {
- return () -> {
- if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
- (k) -> true)) {
- nodeProcessed.incrementAndGet();
- LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
- datanodeDetails.getUuid());
- for (ContainerInfo info : reports.getReportsList()) {
- containerProcessedCount.incrementAndGet();
- LOG.debug("Total Containers processed: {} Container Name: {}",
- containerProcessedCount.get(), info.getContainerID());
-
- // Update the container map with count + 1 if the key exists or
- // update the map with 1. Since this is a concurrentMap the
- // computation and update is atomic.
- containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
- }
- }
- };
- }
-
- /**
- * Filter the containers based on specific rules.
- *
- * @param predicate -- Predicate to filter by
- * @return A list of map entries.
- */
- public List<Map.Entry<Long, Integer>> filterContainer(
- Predicate<Map.Entry<Long, Integer>> predicate) {
- return containerCountMap.entrySet().stream()
- .filter(predicate).collect(Collectors.toList());
- }
-
- /**
- * Used only for testing, calling this will abort container report
- * processing. This is very dangerous call and should not be made by any users
- */
- @VisibleForTesting
- public void setDoneProcessing() {
- nodeProcessed.set(nodeCount.get());
- }
-
- /**
- * Returns the pool name.
- *
- * @return Name of the pool.
- */
- String getPoolName() {
- return pool.getPoolName();
- }
-
- public void finalizeReconciliation() {
- status = ProgressStatus.Done;
- //TODO: Add finalizing logic. This is where actual reconciliation happens.
- }
-
- /**
- * Current status of the computing replication status.
- */
- public enum ProgressStatus {
- InProgress, Done, Error
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index ef28aa7..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
- private final String poolName;
- private long lastProcessedTime;
- private AtomicLong totalProcessedCount;
-
- /**
- * Constructs a periodic pool.
- *
- * @param poolName - Name of the pool
- */
- public PeriodicPool(String poolName) {
- this.poolName = poolName;
- lastProcessedTime = 0;
- totalProcessedCount = new AtomicLong(0);
- }
-
- /**
- * Get pool Name.
- * @return PoolName
- */
- public String getPoolName() {
- return poolName;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(PeriodicPool o) {
- return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- PeriodicPool that = (PeriodicPool) o;
-
- return poolName.equals(that.poolName);
- }
-
- @Override
- public int hashCode() {
- return poolName.hashCode();
- }
-
- /**
- * Returns the Total Times we have processed this pool.
- *
- * @return processed count.
- */
- public long getTotalProcessedCount() {
- return totalProcessedCount.get();
- }
-
- /**
- * Gets the last time we processed this pool.
- * @return time in milliseconds
- */
- public long getLastProcessedTime() {
- return this.lastProcessedTime;
- }
-
-
- /**
- * Sets the last processed time.
- *
- * @param lastProcessedTime - Long in milliseconds.
- */
-
- public void setLastProcessedTime(long lastProcessedTime) {
- this.lastProcessedTime = lastProcessedTime;
- }
-
- /*
- * Increments the total processed count.
- */
- public void incTotalProcessedCount() {
- this.totalProcessedCount.incrementAndGet();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
deleted file mode 100644
index 7bbe2ef..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 4392633..72d7e94 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,12 +124,6 @@ public interface NodeManager extends StorageContainerNodeProtocol,
SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
/**
- * Returns the NodePoolManager associated with the NodeManager.
- * @return NodePoolManager
- */
- NodePoolManager getNodePoolManager();
-
- /**
* Wait for the heartbeat is processed by NodeManager.
* @return true if heartbeat has been processed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
deleted file mode 100644
index 46faf9ca..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
- /**
- * Add a node to a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- */
- void addNode(String pool, DatanodeDetails node) throws IOException;
-
- /**
- * Remove a node from a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- * @throws SCMException
- */
- void removeNode(String pool, DatanodeDetails node)
- throws SCMException;
-
- /**
- * Get a list of known node pools.
- * @return a list of known node pool names or an empty list if not node pool
- * is defined.
- */
- List<String> getNodePools();
-
- /**
- * Get all nodes of a node pool given the name of the node pool.
- * @param pool - name of the node pool.
- * @return a list of datanode ids or an empty list if the node pool was not
- * found.
- */
- List<DatanodeDetails> getNodes(String pool);
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeDetails - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- */
- String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fc8b013..adca8ea 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import com.google.protobuf.GeneratedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -159,7 +158,6 @@ public class SCMNodeManager
private ObjectName nmInfoBean;
// Node pool manager.
- private final SCMNodePoolManager nodePoolManager;
private final StorageContainerManager scmManager;
public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -210,7 +208,6 @@ public class SCMNodeManager
registerMXBean();
- this.nodePoolManager = new SCMNodePoolManager(conf);
this.scmManager = scmManager;
}
@@ -682,7 +679,6 @@ public class SCMNodeManager
@Override
public void close() throws IOException {
unregisterMXBean();
- nodePoolManager.close();
executorService.shutdown();
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -760,20 +756,6 @@ public class SCMNodeManager
LOG.info("Leaving startup chill mode.");
}
- // TODO: define node pool policy for non-default node pool.
- // For now, all nodes are added to the "DefaultNodePool" upon registration
- // if it has not been added to any node pool yet.
- try {
- if (nodePoolManager.getNodePool(datanodeDetails) == null) {
- nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
- datanodeDetails);
- }
- } catch (IOException e) {
- // TODO: make sure registration failure is handled correctly.
- return RegisteredCommand.newBuilder()
- .setErrorCode(ErrorCode.errorNodeNotPermitted)
- .build();
- }
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
LOG.info("Data node with ID: {} Registered.",
@@ -860,11 +842,6 @@ public class SCMNodeManager
}
@Override
- public NodePoolManager getNodePoolManager() {
- return nodePoolManager;
- }
-
- @Override
public Map<String, Integer> getNodeCount() {
Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
for(NodeState state : NodeState.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index faf330e..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMNodePoolManager.class);
- private static final List<DatanodeDetails> EMPTY_NODE_LIST =
- new ArrayList<>();
- private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
- public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
- // DB that saves the node to node pool mapping.
- private MetadataStore nodePoolStore;
-
- // In-memory node pool to nodes mapping
- private HashMap<String, Set<DatanodeDetails>> nodePools;
-
- // Read-write lock for nodepool operations
- private ReadWriteLock lock;
-
- /**
- * Construct SCMNodePoolManager class that manages node to node pool mapping.
- * @param conf - configuration.
- * @throws IOException
- */
- public SCMNodePoolManager(final OzoneConfiguration conf)
- throws IOException {
- final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- File metaDir = getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
- File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
- nodePoolStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(nodePoolDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
- nodePools = new HashMap<>();
- lock = new ReentrantReadWriteLock();
- init();
- }
-
- /**
- * Initialize the in-memory store based on persist store from level db.
- * No lock is needed as init() is only invoked by constructor.
- * @throws SCMException
- */
- private void init() throws SCMException {
- try {
- nodePoolStore.iterate(null, (key, value) -> {
- try {
- DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
- HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
- String poolName = DFSUtil.bytes2String(value);
-
- Set<DatanodeDetails> nodePool = null;
- if (nodePools.containsKey(poolName)) {
- nodePool = nodePools.get(poolName);
- } else {
- nodePool = new HashSet<>();
- nodePools.put(poolName, nodePool);
- }
- nodePool.add(nodeId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding node: {} to node pool: {}",
- nodeId, poolName);
- }
- } catch (IOException e) {
- LOG.warn("Can't add a datanode to node pool, continue next...");
- }
- return true;
- });
- } catch (IOException e) {
- LOG.error("Loading node pool error " + e);
- throw new SCMException("Failed to load node pool",
- FAILED_TO_LOAD_NODEPOOL);
- }
- }
-
- /**
- * Add a datanode to a node pool.
- * @param pool - name of the node pool.
- * @param node - name of the datanode.
- */
- @Override
- public void addNode(final String pool, final DatanodeDetails node)
- throws IOException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // add to the persistent store
- nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
- DFSUtil.string2Bytes(pool));
-
- // add to the in-memory store
- Set<DatanodeDetails> nodePool = null;
- if (nodePools.containsKey(pool)) {
- nodePool = nodePools.get(pool);
- } else {
- nodePool = new HashSet<DatanodeDetails>();
- nodePools.put(pool, nodePool);
- }
- nodePool.add(node);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Remove a datanode from a node pool.
- * @param pool - name of the node pool.
- * @param node - datanode id.
- * @throws SCMException
- */
- @Override
- public void removeNode(final String pool, final DatanodeDetails node)
- throws SCMException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // Remove from the persistent store
- byte[] kName = node.getProtoBufMessage().toByteArray();
- byte[] kData = nodePoolStore.get(kName);
- if (kData == null) {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL);
- }
- nodePoolStore.delete(kName);
-
- // Remove from the in-memory store
- if (nodePools.containsKey(pool)) {
- Set<DatanodeDetails> nodePool = nodePools.get(pool);
- nodePool.remove(node);
- } else {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL);
- }
- } catch (IOException e) {
- throw new SCMException("Failed to remove node " + node.toString()
- + " from node pool " + pool, e,
- SCMException.ResultCodes.IO_EXCEPTION);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Get all the node pools.
- * @return all the node pools.
- */
- @Override
- public List<String> getNodePools() {
- lock.readLock().lock();
- try {
- if (!nodePools.isEmpty()) {
- return nodePools.keySet().stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODEPOOL_LIST;
- }
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Get all datanodes of a specific node pool.
- * @param pool - name of the node pool.
- * @return all datanodes of the specified node pool.
- */
- @Override
- public List<DatanodeDetails> getNodes(final String pool) {
- Preconditions.checkNotNull(pool, "pool name is null");
- if (nodePools.containsKey(pool)) {
- return nodePools.get(pool).stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODE_LIST;
- }
- }
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeDetails - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- * TODO: Put this in a in-memory map if performance is an issue.
- */
- @Override
- public String getNodePool(final DatanodeDetails datanodeDetails)
- throws SCMException {
- Preconditions.checkNotNull(datanodeDetails, "node is null");
- try {
- byte[] result = nodePoolStore.get(
- datanodeDetails.getProtoBufMessage().toByteArray());
- return result == null ? null : DFSUtil.bytes2String(result);
- } catch (IOException e) {
- throw new SCMException("Failed to get node pool for node "
- + datanodeDetails.toString(), e,
- SCMException.ResultCodes.IO_EXCEPTION);
- }
- }
-
- /**
- * Close node pool level db store.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- nodePoolStore.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 8c59462..80b5d6e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -273,11 +272,6 @@ public class MockNodeManager implements NodeManager {
return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
}
- @Override
- public NodePoolManager getNodePoolManager() {
- return Mockito.mock(NodePoolManager.class);
- }
-
/**
* Used for testing.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
deleted file mode 100644
index 8f412de..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.commons.collections.ListUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
- .SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test for SCM node pool manager.
- */
-public class TestSCMNodePoolManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestSCMNodePoolManager.class);
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- private final File testDir = PathUtils.getTestDir(
- TestSCMNodePoolManager.class);
-
- SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
- throws IOException {
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
- testDir.getAbsolutePath());
- conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
- return new SCMNodePoolManager(conf);
- }
-
- /**
- * Test default node pool.
- *
- * @throws IOException
- */
- @Test
- public void testDefaultNodePool() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- try {
- final String defaultPool = "DefaultPool";
- NodePoolManager npMgr = createNodePoolManager(conf);
-
- final int nodeCount = 4;
- final List<DatanodeDetails> nodes = TestUtils
- .getListOfDatanodeDetails(nodeCount);
- assertEquals(0, npMgr.getNodePools().size());
- for (DatanodeDetails node: nodes) {
- npMgr.addNode(defaultPool, node);
- }
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
-
- DatanodeDetails nodeRemoved = nodes.remove(2);
- npMgr.removeNode(defaultPool, nodeRemoved);
- List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
- assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
-
- List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
- assertEquals(0, nonExistSet.size());
- } finally {
- FileUtil.fullyDelete(testDir);
- }
- }
-
-
- /**
- * Test default node pool reload.
- *
- * @throws IOException
- */
- @Test
- public void testDefaultNodePoolReload() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- final String defaultPool = "DefaultPool";
- final int nodeCount = 4;
- final List<DatanodeDetails> nodes = TestUtils
- .getListOfDatanodeDetails(nodeCount);
-
- try {
- try {
- SCMNodePoolManager npMgr = createNodePoolManager(conf);
- assertEquals(0, npMgr.getNodePools().size());
- for (DatanodeDetails node : nodes) {
- npMgr.addNode(defaultPool, node);
- }
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
- npMgr.close();
- } finally {
- LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
- " and close.");
- }
-
- // try reload with a new NodePoolManager instance
- try {
- SCMNodePoolManager npMgr = createNodePoolManager(conf);
- List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
- assertEquals(nodeCount, nodesRetrieved.size());
- assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
- } finally {
- LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
- }
- } finally {
- FileUtil.fullyDelete(testDir);
- }
- }
-
- /**
- * Compare and verify that two datanode lists are equal.
- * @param list1 - datanode list 1.
- * @param list2 - datanode list 2.
- */
- private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
- List<DatanodeDetails> list2) {
- assertEquals(list1.size(), list2.size());
- Collections.sort(list1);
- Collections.sort(list2);
- assertTrue(ListUtils.isEqualList(list1, list2));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 072d821..1a4dcd7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -201,10 +200,6 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
- @Override
- public NodePoolManager getNodePoolManager() {
- return Mockito.mock(NodePoolManager.class);
- }
/**
* Wait for the heartbeat is processed by NodeManager.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
deleted file mode 100644
index ffcd752..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.testutils;
-
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodePoolManager;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Pool Manager replication mock.
- */
-public class ReplicationNodePoolManagerMock implements NodePoolManager {
-
- private final Map<DatanodeDetails, String> nodeMemberShip;
-
- /**
- * A node pool manager for testing.
- */
- public ReplicationNodePoolManagerMock() {
- nodeMemberShip = new HashMap<>();
- }
-
- /**
- * Add a node to a node pool.
- *
- * @param pool - name of the node pool.
- * @param node - data node.
- */
- @Override
- public void addNode(String pool, DatanodeDetails node) {
- nodeMemberShip.put(node, pool);
- }
-
- /**
- * Remove a node from a node pool.
- *
- * @param pool - name of the node pool.
- * @param node - data node.
- * @throws SCMException
- */
- @Override
- public void removeNode(String pool, DatanodeDetails node)
- throws SCMException {
- nodeMemberShip.remove(node);
-
- }
-
- /**
- * Get a list of known node pools.
- *
- * @return a list of known node pool names or an empty list if not node pool
- * is defined.
- */
- @Override
- public List<String> getNodePools() {
- Set<String> poolSet = new HashSet<>();
- for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
- poolSet.add(entry.getValue());
- }
- return new ArrayList<>(poolSet);
-
- }
-
- /**
- * Get all nodes of a node pool given the name of the node pool.
- *
- * @param pool - name of the node pool.
- * @return a list of datanode ids or an empty list if the node pool was not
- * found.
- */
- @Override
- public List<DatanodeDetails> getNodes(String pool) {
- Set<DatanodeDetails> datanodeSet = new HashSet<>();
- for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
- if (entry.getValue().equals(pool)) {
- datanodeSet.add(entry.getKey());
- }
- }
- return new ArrayList<>(datanodeSet);
- }
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- *
- * @param datanodeDetails DatanodeDetails.
- * @return node pool name if it has been assigned. null if the node has not
- * been assigned to any node pool yet.
- */
- @Override
- public String getNodePool(DatanodeDetails datanodeDetails) {
- return nodeMemberShip.get(datanodeDetails);
- }
-
- /**
- * Closes this stream and releases any system resources associated
- * with it. If the stream is already closed then invoking this
- * method has no effect.
- * <p>
- * <p> As noted in {@link AutoCloseable#close()}, cases where the
- * close may fail require careful attention. It is strongly advised
- * to relinquish the underlying resources and to internally
- * <em>mark</em> the {@code Closeable} as closed, prior to throwing
- * the {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 4d70af8..b4ed2b1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,12 +51,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* This class tests the CLI that transforms container into SQLite DB files.
@@ -177,34 +174,6 @@ public class TestContainerSQLCli {
}
@Test
- public void testConvertNodepoolDB() throws Exception {
- String dbOutPath = GenericTestUtils.getTempPath(
- UUID.randomUUID() + "/out_sql.db");
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
- String dbPath = dbRootPath + "/" + NODEPOOL_DB;
- String[] args = {"-p", dbPath, "-o", dbOutPath};
-
- cli.run(args);
-
- // verify the sqlite db
- HashMap<String, String> expectedPool = new HashMap<>();
- for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
- expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
- }
- Connection conn = connectDB(dbOutPath);
- String sql = "SELECT * FROM nodePool";
- ResultSet rs = executeQuery(conn, sql);
- while(rs.next()) {
- String datanodeUUID = rs.getString("datanodeUUID");
- String poolName = rs.getString("poolName");
- assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
- }
- assertEquals(0, expectedPool.size());
-
- Files.delete(Paths.get(dbOutPath));
- }
-
- @Test
public void testConvertContainerDB() throws Exception {
String dbOutPath = GenericTestUtils.getTempPath(
UUID.randomUUID() + "/out_sql.db");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56a4cdb9/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index 2bd43fb..edc0d7b 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.scm.cli;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
@@ -60,13 +59,11 @@ import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
/**
@@ -111,15 +108,6 @@ public class SQLCLI extends Configured implements Tool {
private static final String INSERT_CONTAINER_MEMBERS =
"INSERT INTO containerMembers (containerName, datanodeUUID) " +
"VALUES (\"%s\", \"%s\")";
- // for nodepool.db
- private static final String CREATE_NODE_POOL =
- "CREATE TABLE nodePool (" +
- "datanodeUUID TEXT NOT NULL," +
- "poolName TEXT NOT NULL," +
- "PRIMARY KEY(datanodeUUID, poolName))";
- private static final String INSERT_NODE_POOL =
- "INSERT INTO nodePool (datanodeUUID, poolName) " +
- "VALUES (\"%s\", \"%s\")";
// and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
// for openContainer.db
private static final String CREATE_OPEN_CONTAINER =
@@ -285,9 +273,6 @@ public class SQLCLI extends Configured implements Tool {
if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
LOG.info("Converting container DB");
convertContainerDB(dbPath, outPath);
- } else if (dbName.toString().equals(NODEPOOL_DB)) {
- LOG.info("Converting node pool DB");
- convertNodePoolDB(dbPath, outPath);
} else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
LOG.info("Converting open container DB");
convertOpenContainerDB(dbPath, outPath);
@@ -543,66 +528,7 @@ public class SQLCLI extends Configured implements Tool {
}
LOG.info("Insertion completed.");
}
- /**
- * Converts nodePool.db to sqlite. The schema of sql db:
- * two tables, nodePool and datanodeInfo (the same datanode Info as for
- * container.db).
- *
- * nodePool
- * ---------------------------------------------------------
- * datanodeUUID* | poolName*
- * ---------------------------------------------------------
- *
- * datanodeInfo:
- * ---------------------------------------------------------
- * hostname | datanodeUUid* | xferPort | ipcPort
- * ---------------------------------------------------------
- *
- * --------------------------------
- * |containerPort
- * --------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create table for sql node pool db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_NODE_POOL);
- executeSQL(conn, CREATE_DATANODE_INFO);
- dbStore.iterate(null, (key, value) -> {
- DatanodeDetails nodeId = DatanodeDetails
- .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
- .PARSER.parseFrom(key));
- String blockPool = DFSUtil.bytes2String(value);
- try {
- insertNodePoolDB(conn, blockPool, nodeId);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- private void insertNodePoolDB(Connection conn, String blockPool,
- DatanodeDetails datanodeDetails) throws SQLException {
- String insertNodePool = String.format(INSERT_NODE_POOL,
- datanodeDetails.getUuidString(), blockPool);
- executeSQL(conn, insertNodePool);
-
- String insertDatanodeDetails = String
- .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
- datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
- datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
- .getValue());
- executeSQL(conn, insertDatanodeDetails);
- }
/**
* Convert openContainer.db to sqlite db file. This is rather simple db,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
deleted file mode 100644
index 3156eb2..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-
-import java.io.IOException;
-
-/**
- * Ozone contract test for ROOT directory operations.
- */
-public class ITestOzoneContractRootDir extends
- AbstractContractRootDirectoryTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
deleted file mode 100644
index c4bc0ff..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering file seek.
- */
-public class ITestOzoneContractSeek extends AbstractContractSeekTest {
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
deleted file mode 100644
index 8417e46..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.junit.Assert;
-
-import java.io.IOException;
-
-/**
- * The contract of Ozone: only enabled if the test bucket is provided.
- */
-class OzoneContract extends AbstractFSContract {
-
- private static MiniOzoneCluster cluster;
- private static StorageHandler storageHandler;
- private static final String CONTRACT_XML = "contract/ozone.xml";
-
- OzoneContract(Configuration conf) {
- super(conf);
- //insert the base features
- addConfResource(CONTRACT_XML);
- }
-
- @Override
- public String getScheme() {
- return OzoneConsts.OZONE_URI_SCHEME;
- }
-
- @Override
- public Path getTestPath() {
- Path path = new Path("/test");
- return path;
- }
-
- public static void createCluster() throws IOException {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.addResource(CONTRACT_XML);
-
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- try {
- cluster.waitForClusterToBeReady();
- } catch (Exception e) {
- throw new IOException(e);
- }
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- }
-
- private void copyClusterConfigs(String configKey) {
- getConf().set(configKey, cluster.getConf().get(configKey));
- }
-
- @Override
- public FileSystem getTestFileSystem() throws IOException {
- //assumes cluster is not null
- Assert.assertNotNull("cluster not created", cluster);
-
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
-
-
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- try {
- storageHandler.createVolume(volumeArgs);
- storageHandler.createBucket(bucketArgs);
- } catch (OzoneException e) {
- throw new IOException(e.getMessage());
- }
-
- String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
- getConf().set("fs.defaultFS", uri);
- copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
- copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
- return FileSystem.get(getConf());
- }
-
- public static void destroyCluster() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- cluster = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml b/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
deleted file mode 100644
index fe2075c..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml
+++ /dev/null
@@ -1,113 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration>
- <!--
- Ozone is a blobstore, with very different behavior than a classic filesystem.
- -->
-
- <property>
- <name>fs.contract.test.root-tests-enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.test.random-seek-count</name>
- <value>10</value>
- </property>
-
- <property>
- <name>fs.contract.is-blobstore</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.create-visibility-delayed</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.is-case-sensitive</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-returns-false-if-source-missing</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rename-remove-dest-if-empty-dir</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-append</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-directory-delete</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-atomic-rename</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-block-locality</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-concat</name>
- <value>false</value>
- </property>
-
- <property>
- <name>fs.contract.supports-getfilestatus</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-seek-on-closed-file</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.rejects-seek-past-eof</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-strict-exceptions</name>
- <value>true</value>
- </property>
-
- <property>
- <name>fs.contract.supports-unix-permissions</name>
- <value>false</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
deleted file mode 100644
index 3bf1619..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
-
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-
-# for debugging low level Ozone operations, uncomment this line
-# log4j.logger.org.apache.hadoop.ozone=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 21cc7ce..42ce94c 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -192,20 +192,5 @@
</plugins>
</build>
</profile>
-
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <scope>compile</scope>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index f421e58..dca59d3 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -67,15 +67,4 @@
</plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>hadoop-ozone</module>
- </modules>
- </profile>
- </profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: Revert "HDDS-194. Remove
NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index edc0d7b..2bd43fb 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.scm.cli;
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
+import com.google.protobuf.ByteString;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
@@ -59,11 +60,13 @@ import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
+import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
/**
@@ -108,6 +111,15 @@ public class SQLCLI extends Configured implements Tool {
private static final String INSERT_CONTAINER_MEMBERS =
"INSERT INTO containerMembers (containerName, datanodeUUID) " +
"VALUES (\"%s\", \"%s\")";
+ // for nodepool.db
+ private static final String CREATE_NODE_POOL =
+ "CREATE TABLE nodePool (" +
+ "datanodeUUID TEXT NOT NULL," +
+ "poolName TEXT NOT NULL," +
+ "PRIMARY KEY(datanodeUUID, poolName))";
+ private static final String INSERT_NODE_POOL =
+ "INSERT INTO nodePool (datanodeUUID, poolName) " +
+ "VALUES (\"%s\", \"%s\")";
// and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
// for openContainer.db
private static final String CREATE_OPEN_CONTAINER =
@@ -273,6 +285,9 @@ public class SQLCLI extends Configured implements Tool {
if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
LOG.info("Converting container DB");
convertContainerDB(dbPath, outPath);
+ } else if (dbName.toString().equals(NODEPOOL_DB)) {
+ LOG.info("Converting node pool DB");
+ convertNodePoolDB(dbPath, outPath);
} else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
LOG.info("Converting open container DB");
convertOpenContainerDB(dbPath, outPath);
@@ -528,7 +543,66 @@ public class SQLCLI extends Configured implements Tool {
}
LOG.info("Insertion completed.");
}
+ /**
+ * Converts nodePool.db to sqlite. The schema of sql db:
+ * two tables, nodePool and datanodeInfo (the same datanode Info as for
+ * container.db).
+ *
+ * nodePool
+ * ---------------------------------------------------------
+ * datanodeUUID* | poolName*
+ * ---------------------------------------------------------
+ *
+ * datanodeInfo:
+ * ---------------------------------------------------------
+ * hostname | datanodeUUid* | xferPort | ipcPort
+ * ---------------------------------------------------------
+ *
+ * --------------------------------
+ * |containerPort
+ * --------------------------------
+ *
+ * @param dbPath path to container db.
+ * @param outPath path to output sqlite
+ * @throws IOException throws exception.
+ */
+ private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
+ LOG.info("Create table for sql node pool db.");
+ File dbFile = dbPath.toFile();
+ try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf).setDbFile(dbFile).build();
+ Connection conn = connectDB(outPath.toString())) {
+ executeSQL(conn, CREATE_NODE_POOL);
+ executeSQL(conn, CREATE_DATANODE_INFO);
+ dbStore.iterate(null, (key, value) -> {
+ DatanodeDetails nodeId = DatanodeDetails
+ .getFromProtoBuf(HddsProtos.DatanodeDetailsProto
+ .PARSER.parseFrom(key));
+ String blockPool = DFSUtil.bytes2String(value);
+ try {
+ insertNodePoolDB(conn, blockPool, nodeId);
+ return true;
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ });
+ }
+ }
+
+ private void insertNodePoolDB(Connection conn, String blockPool,
+ DatanodeDetails datanodeDetails) throws SQLException {
+ String insertNodePool = String.format(INSERT_NODE_POOL,
+ datanodeDetails.getUuidString(), blockPool);
+ executeSQL(conn, insertNodePool);
+
+ String insertDatanodeDetails = String
+ .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
+ datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
+ datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
+ .getValue());
+ executeSQL(conn, insertDatanodeDetails);
+ }
/**
* Convert openContainer.db to sqlite db file. This is rather simple db,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-193. Make Datanode heartbeat
dispatcher in SCM event based. Contributed by Elek, Marton.
Posted by gi...@apache.org.
HDDS-193. Make Datanode heartbeat dispatcher in SCM event based.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8752a485
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8752a485
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8752a485
Branch: refs/heads/HADOOP-15461
Commit: 8752a48564028cb5892c19e29d4e5b984d70c076
Parents: 1893271
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Jun 27 14:18:25 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 14:18:25 2018 -0700
----------------------------------------------------------------------
.../server/SCMDatanodeHeartbeatDispatcher.java | 126 +++++++++
.../scm/server/SCMDatanodeProtocolServer.java | 14 +-
.../scm/server/StorageContainerManager.java | 4 +-
.../SCMDatanodeContainerReportHandler.java | 76 ------
.../report/SCMDatanodeHeartbeatDispatcher.java | 189 --------------
.../report/SCMDatanodeNodeReportHandler.java | 43 ----
.../server/report/SCMDatanodeReportHandler.java | 83 ------
.../report/SCMDatanodeReportHandlerFactory.java | 82 ------
.../hdds/scm/server/report/package-info.java | 57 -----
.../TestSCMDatanodeHeartbeatDispatcher.java | 119 +++++++++
.../TestSCMDatanodeContainerReportHandler.java | 34 ---
.../TestSCMDatanodeHeartbeatDispatcher.java | 138 ----------
.../TestSCMDatanodeNodeReportHandler.java | 36 ---
.../TestSCMDatanodeReportHandlerFactory.java | 51 ----
.../hdds/scm/server/report/package-info.java | 21 --
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 253 -------------------
16 files changed, 254 insertions(+), 1072 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
new file mode 100644
index 0000000..36f10a9
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
+
+import com.google.protobuf.GeneratedMessage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class is responsible for dispatching heartbeat from datanode to
+ * appropriate EventHandler at SCM.
+ */
+public final class SCMDatanodeHeartbeatDispatcher {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SCMDatanodeHeartbeatDispatcher.class);
+
+ private EventPublisher eventPublisher;
+
+ public static final TypedEvent<NodeReportFromDatanode> NODE_REPORT =
+ new TypedEvent<>(NodeReportFromDatanode.class);
+
+ public static final TypedEvent<ContainerReportFromDatanode> CONTAINER_REPORT =
+ new TypedEvent<ContainerReportFromDatanode>(ContainerReportFromDatanode.class);
+
+ public SCMDatanodeHeartbeatDispatcher(EventPublisher eventPublisher) {
+ this.eventPublisher = eventPublisher;
+ }
+
+
+ /**
+ * Dispatches heartbeat to registered event handlers.
+ *
+ * @param heartbeat heartbeat to be dispatched.
+ */
+ public void dispatch(SCMHeartbeatRequestProto heartbeat) {
+ DatanodeDetails datanodeDetails =
+ DatanodeDetails.getFromProtoBuf(heartbeat.getDatanodeDetails());
+
+ if (heartbeat.hasNodeReport()) {
+ eventPublisher.fireEvent(NODE_REPORT,
+ new NodeReportFromDatanode(datanodeDetails,
+ heartbeat.getNodeReport()));
+ }
+
+ if (heartbeat.hasContainerReport()) {
+ eventPublisher.fireEvent(CONTAINER_REPORT,
+ new ContainerReportFromDatanode(datanodeDetails,
+ heartbeat.getContainerReport()));
+
+ }
+ }
+
+ /**
+ * Wrapper class for events with the datanode origin.
+ */
+ public static class ReportFromDatanode<T extends GeneratedMessage> {
+
+ private final DatanodeDetails datanodeDetails;
+
+ private final T report;
+
+ public ReportFromDatanode(DatanodeDetails datanodeDetails, T report) {
+ this.datanodeDetails = datanodeDetails;
+ this.report = report;
+ }
+
+ public DatanodeDetails getDatanodeDetails() {
+ return datanodeDetails;
+ }
+
+ public T getReport() {
+ return report;
+ }
+ }
+
+ /**
+ * Node report event payload with origin.
+ */
+ public static class NodeReportFromDatanode
+ extends ReportFromDatanode<NodeReportProto> {
+
+ public NodeReportFromDatanode(DatanodeDetails datanodeDetails,
+ NodeReportProto report) {
+ super(datanodeDetails, report);
+ }
+ }
+
+ /**
+ * Container report event payload with origin.
+ */
+ public static class ContainerReportFromDatanode
+ extends ReportFromDatanode<ContainerReportsProto> {
+
+ public ContainerReportFromDatanode(DatanodeDetails datanodeDetails,
+ ContainerReportsProto report) {
+ super(datanodeDetails, report);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index eb5ce1a..56b0719 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -73,7 +73,7 @@ import static org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.server.report.SCMDatanodeHeartbeatDispatcher;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -122,14 +122,19 @@ public class SCMDatanodeProtocolServer implements
private final SCMDatanodeHeartbeatDispatcher heartbeatDispatcher;
public SCMDatanodeProtocolServer(final OzoneConfiguration conf,
- StorageContainerManager scm) throws IOException {
+ StorageContainerManager scm, EventPublisher eventPublisher)
+ throws IOException {
Preconditions.checkNotNull(scm, "SCM cannot be null");
+ Preconditions.checkNotNull(eventPublisher, "EventPublisher cannot be null");
+
this.scm = scm;
final int handlerCount =
conf.getInt(OZONE_SCM_HANDLER_COUNT_KEY,
OZONE_SCM_HANDLER_COUNT_DEFAULT);
+ heartbeatDispatcher = new SCMDatanodeHeartbeatDispatcher(eventPublisher);
+
RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
ProtobufRpcEngine.class);
BlockingService dnProtoPbService =
@@ -155,10 +160,6 @@ public class SCMDatanodeProtocolServer implements
conf, OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr,
datanodeRpcServer);
- heartbeatDispatcher = SCMDatanodeHeartbeatDispatcher.newBuilder(conf, scm)
- .addHandlerFor(NodeReportProto.class)
- .addHandlerFor(ContainerReportsProto.class)
- .build();
}
public void start() {
@@ -319,7 +320,6 @@ public class SCMDatanodeProtocolServer implements
try {
LOG.info("Stopping the RPC server for DataNodes");
datanodeRpcServer.stop();
- heartbeatDispatcher.shutdown();
} catch (Exception ex) {
LOG.error(" datanodeRpcServer stop failed.", ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 5725d23..568a86a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.common.StorageInfo;
-import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
@@ -182,7 +181,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
scmAdminUsernames.add(scmUsername);
}
- datanodeProtocolServer = new SCMDatanodeProtocolServer(conf, this);
+ datanodeProtocolServer = new SCMDatanodeProtocolServer(conf, this,
+ eventQueue);
blockProtocolServer = new SCMBlockProtocolServer(conf, this);
clientProtocolServer = new SCMClientProtocolServer(conf, this);
httpServer = new StorageContainerManagerHttpServer(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
deleted file mode 100644
index 00ce94d..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Handler for Datanode Container Report.
- */
-public class SCMDatanodeContainerReportHandler extends
- SCMDatanodeReportHandler<ContainerReportsProto> {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SCMDatanodeContainerReportHandler.class);
-
- @Override
- public void processReport(DatanodeDetails datanodeDetails,
- ContainerReportsProto report) throws IOException {
- LOG.trace("Processing container report from {}.", datanodeDetails);
- updateContainerReportMetrics(datanodeDetails, report);
- getSCM().getScmContainerManager()
- .processContainerReports(datanodeDetails, report);
- }
-
- /**
- * Updates container report metrics in SCM.
- *
- * @param datanodeDetails Datanode Information
- * @param reports Container Reports
- */
- private void updateContainerReportMetrics(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports) {
- ContainerStat newStat = new ContainerStat();
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports
- .getReportsList()) {
- newStat.add(new ContainerStat(info.getSize(), info.getUsed(),
- info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(),
- info.getReadCount(), info.getWriteCount()));
- }
- // update container metrics
- StorageContainerManager.getMetrics().setLastContainerStat(newStat);
-
- // Update container stat entry, this will trigger a removal operation if it
- // exists in cache.
- String datanodeUuid = datanodeDetails.getUuidString();
- getSCM().getContainerReportCache().put(datanodeUuid, newStat);
- // update global view container metrics
- StorageContainerManager.getMetrics().incrContainerStat(newStat);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
deleted file mode 100644
index d50edff..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-/**
- * This class is responsible for dispatching heartbeat from datanode to
- * appropriate ReportHandlers at SCM.
- * Only one handler per report is supported now, it's very easy to support
- * multiple handlers for a report.
- */
-public final class SCMDatanodeHeartbeatDispatcher {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SCMDatanodeHeartbeatDispatcher.class);
-
- /**
- * This stores Report to Handler mapping.
- */
- private final Map<Class<? extends GeneratedMessage>,
- SCMDatanodeReportHandler<? extends GeneratedMessage>> handlers;
-
- /**
- * Executor service which will be used for processing reports.
- */
- private final ExecutorService executorService;
-
- /**
- * Constructs SCMDatanodeHeartbeatDispatcher instance with the given
- * handlers.
- *
- * @param handlers report to report handler mapping
- */
- private SCMDatanodeHeartbeatDispatcher(Map<Class<? extends GeneratedMessage>,
- SCMDatanodeReportHandler<? extends GeneratedMessage>> handlers) {
- this.handlers = handlers;
- this.executorService = HadoopExecutors.newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("SCMDatanode Heartbeat Dispatcher Thread - %d")
- .build());
- }
-
- /**
- * Dispatches heartbeat to registered handlers.
- *
- * @param heartbeat heartbeat to be dispatched.
- */
- public void dispatch(SCMHeartbeatRequestProto heartbeat) {
- DatanodeDetails datanodeDetails = DatanodeDetails
- .getFromProtoBuf(heartbeat.getDatanodeDetails());
- if (heartbeat.hasNodeReport()) {
- processReport(datanodeDetails, heartbeat.getNodeReport());
- }
- if (heartbeat.hasContainerReport()) {
- processReport(datanodeDetails, heartbeat.getContainerReport());
- }
- }
-
- /**
- * Invokes appropriate ReportHandler and submits the task to executor
- * service for processing.
- *
- * @param datanodeDetails Datanode Information
- * @param report Report to be processed
- */
- @SuppressWarnings("unchecked")
- private void processReport(DatanodeDetails datanodeDetails,
- GeneratedMessage report) {
- executorService.submit(() -> {
- try {
- SCMDatanodeReportHandler handler = handlers.get(report.getClass());
- handler.processReport(datanodeDetails, report);
- } catch (IOException ex) {
- LOG.error("Exception wile processing report {}, from {}",
- report.getClass(), datanodeDetails, ex);
- }
- });
- }
-
- /**
- * Shuts down SCMDatanodeHeartbeatDispatcher.
- */
- public void shutdown() {
- executorService.shutdown();
- }
-
- /**
- * Returns a new Builder to construct {@link SCMDatanodeHeartbeatDispatcher}.
- *
- * @param conf Configuration to be used by SCMDatanodeHeartbeatDispatcher
- * @param scm {@link StorageContainerManager} instance to be used by report
- * handlers
- *
- * @return {@link SCMDatanodeHeartbeatDispatcher.Builder} instance
- */
- public static Builder newBuilder(Configuration conf,
- StorageContainerManager scm) {
- return new Builder(conf, scm);
- }
-
- /**
- * Builder for SCMDatanodeHeartbeatDispatcher.
- */
- public static class Builder {
-
- private final SCMDatanodeReportHandlerFactory reportHandlerFactory;
- private final Map<Class<? extends GeneratedMessage>,
- SCMDatanodeReportHandler<? extends GeneratedMessage>> report2handler;
-
- /**
- * Constructs SCMDatanodeHeartbeatDispatcher.Builder instance.
- *
- * @param conf Configuration object to be used.
- * @param scm StorageContainerManager instance to be used for report
- * handler initialization.
- */
- private Builder(Configuration conf, StorageContainerManager scm) {
- this.report2handler = new HashMap<>();
- this.reportHandlerFactory =
- new SCMDatanodeReportHandlerFactory(conf, scm);
- }
-
- /**
- * Adds new report handler for the given report.
- *
- * @param report Report for which handler has to be added
- *
- * @return Builder
- */
- public Builder addHandlerFor(Class<? extends GeneratedMessage> report) {
- report2handler.put(report, reportHandlerFactory.getHandlerFor(report));
- return this;
- }
-
- /**
- * Associates the given report handler for the given report.
- *
- * @param report Report to be associated with
- * @param handler Handler to be used for the report
- *
- * @return Builder
- */
- public Builder addHandler(Class<? extends GeneratedMessage> report,
- SCMDatanodeReportHandler<? extends GeneratedMessage> handler) {
- report2handler.put(report, handler);
- return this;
- }
-
- /**
- * Builds and returns {@link SCMDatanodeHeartbeatDispatcher} instance.
- *
- * @return SCMDatanodeHeartbeatDispatcher
- */
- public SCMDatanodeHeartbeatDispatcher build() {
- return new SCMDatanodeHeartbeatDispatcher(report2handler);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
deleted file mode 100644
index fb89b02..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Handles Datanode Node Report.
- */
-public class SCMDatanodeNodeReportHandler extends
- SCMDatanodeReportHandler<NodeReportProto> {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SCMDatanodeNodeReportHandler.class);
-
- @Override
- public void processReport(DatanodeDetails datanodeDetails,
- NodeReportProto report) throws IOException {
- LOG.debug("Processing node report from {}.", datanodeDetails);
- //TODO: add logic to process node report.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
deleted file mode 100644
index d338649..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-
-import java.io.IOException;
-
-/**
- * Datanode Report handlers should implement this interface in order to get
- * call back whenever the report is received from datanode.
- *
- * @param <T> Type of report the handler is interested in.
- */
-public abstract class SCMDatanodeReportHandler<T extends GeneratedMessage>
- implements Configurable {
-
- private Configuration config;
- private StorageContainerManager scm;
-
- /**
- * Initializes SCMDatanodeReportHandler and associates it with the given
- * StorageContainerManager instance.
- *
- * @param storageContainerManager StorageContainerManager instance to be
- * associated with.
- */
- public void init(StorageContainerManager storageContainerManager) {
- this.scm = storageContainerManager;
- }
-
- /**
- * Returns the associated StorageContainerManager instance. This will be
- * used by the ReportHandler implementations.
- *
- * @return {@link StorageContainerManager}
- */
- protected StorageContainerManager getSCM() {
- return scm;
- }
-
- @Override
- public void setConf(Configuration conf) {
- this.config = conf;
- }
-
- @Override
- public Configuration getConf() {
- return config;
- }
-
- /**
- * Processes the report received from datanode. Each ReportHandler
- * implementation is responsible for providing the logic to process the
- * report it's interested in.
- *
- * @param datanodeDetails Datanode Information
- * @param report Report to be processed
- *
- * @throws IOException In case of any exception
- */
- abstract void processReport(DatanodeDetails datanodeDetails, T report)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
deleted file mode 100644
index e88495f..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import com.google.protobuf.GeneratedMessage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.util.ReflectionUtils;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-/**
- * Factory class to construct {@link SCMDatanodeReportHandler} given a report.
- */
-public class SCMDatanodeReportHandlerFactory {
-
- private final Configuration conf;
- private final StorageContainerManager scm;
- private final Map<Class<? extends GeneratedMessage>,
- Class<? extends SCMDatanodeReportHandler<? extends GeneratedMessage>>>
- report2handler;
-
- /**
- * Constructs {@link SCMDatanodeReportHandler} instance.
- *
- * @param conf Configuration to be passed to the
- * {@link SCMDatanodeReportHandler}
- */
- public SCMDatanodeReportHandlerFactory(Configuration conf,
- StorageContainerManager scm) {
- this.conf = conf;
- this.scm = scm;
- this.report2handler = new HashMap<>();
-
- report2handler.put(NodeReportProto.class,
- SCMDatanodeNodeReportHandler.class);
- report2handler.put(ContainerReportsProto.class,
- SCMDatanodeContainerReportHandler.class);
- }
-
- /**
- * Returns the SCMDatanodeReportHandler for the corresponding report.
- *
- * @param report report
- *
- * @return report handler
- */
- public SCMDatanodeReportHandler<? extends GeneratedMessage> getHandlerFor(
- Class<? extends GeneratedMessage> report) {
- Class<? extends SCMDatanodeReportHandler<? extends GeneratedMessage>>
- handlerClass = report2handler.get(report);
- if (handlerClass == null) {
- throw new RuntimeException("No handler found for report " + report);
- }
- SCMDatanodeReportHandler<? extends GeneratedMessage> instance =
- ReflectionUtils.newInstance(handlerClass, conf);
- instance.init(scm);
- return instance;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
deleted file mode 100644
index fda3993..0000000
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.server.report;
-/**
- * Handling of all the datanode reports in SCM which are received through
- * heartbeat is done here.
- *
- * SCM Datanode Report Processing State Diagram:
- *
- * SCMDatanode SCMDatanodeHeartbeat SCMDatanodeReport
- * ProtocolServer Dispatcher Handler
- * | | |
- * | | |
- * | construct | |
- * |----------------------->| |
- * | | |
- * | | register |
- * | |<-----------------------|
- * | | |
- * +------------+------------------------+------------------------+--------+
- * | loop | | | |
- * | | | | |
- * | | | | |
- * | heartbeat | | | |
- * - +----------->| | | |
- * | from | heartbeat | | |
- * | Datanode |----------------------->| | |
- * | | | report | |
- * | | |----------------------->| |
- * | | | | |
- * | DN | | | |
- * <-+------------| | | |
- * | commands | | | |
- * | | | | |
- * +------------+------------------------+------------------------+--------+
- * | | |
- * | | |
- * | shutdown | |
- * |----------------------->| |
- * | | |
- * | | |
- * - - -
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
new file mode 100644
index 0000000..326a34b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.NodeReportProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .ContainerReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .NodeReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the behavior of SCMDatanodeHeartbeatDispatcher.
+ */
+public class TestSCMDatanodeHeartbeatDispatcher {
+
+
+ @Test
+ public void testNodeReportDispatcher() throws IOException {
+
+ Configuration conf = new OzoneConfiguration();
+
+ AtomicInteger eventReceived = new AtomicInteger();
+
+ NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
+
+ SCMDatanodeHeartbeatDispatcher dispatcher =
+ new SCMDatanodeHeartbeatDispatcher(new EventPublisher() {
+ @Override
+ public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
+ EVENT_TYPE event, PAYLOAD payload) {
+ Assert.assertEquals(event,
+ SCMDatanodeHeartbeatDispatcher.NODE_REPORT);
+ eventReceived.incrementAndGet();
+ Assert.assertEquals(nodeReport, ((NodeReportFromDatanode)payload).getReport());
+
+ }
+ });
+
+ DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+
+ SCMHeartbeatRequestProto heartbeat =
+ SCMHeartbeatRequestProto.newBuilder()
+ .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+ .setNodeReport(nodeReport)
+ .build();
+ dispatcher.dispatch(heartbeat);
+ Assert.assertEquals(1, eventReceived.get());
+
+
+ }
+
+ @Test
+ public void testContainerReportDispatcher() throws IOException {
+
+ Configuration conf = new OzoneConfiguration();
+
+ AtomicInteger eventReceived = new AtomicInteger();
+
+ ContainerReportsProto containerReport =
+ ContainerReportsProto.getDefaultInstance();
+
+ SCMDatanodeHeartbeatDispatcher dispatcher =
+ new SCMDatanodeHeartbeatDispatcher(new EventPublisher() {
+ @Override
+ public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
+ EVENT_TYPE event, PAYLOAD payload) {
+ Assert.assertEquals(event,
+ SCMDatanodeHeartbeatDispatcher.CONTAINER_REPORT);
+ Assert.assertEquals(containerReport, ((ContainerReportFromDatanode)payload).getReport());
+ eventReceived.incrementAndGet();
+ }
+ });
+
+ DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+
+ SCMHeartbeatRequestProto heartbeat =
+ SCMHeartbeatRequestProto.newBuilder()
+ .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
+ .setContainerReport(containerReport)
+ .build();
+ dispatcher.dispatch(heartbeat);
+ Assert.assertEquals(1, eventReceived.get());
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
deleted file mode 100644
index 776ae88..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify SCMDatanodeContainerReportHandler's behavior.
- */
-public class TestSCMDatanodeContainerReportHandler {
-
- //TODO: add test cases to verify SCMDatanodeContainerReportHandler.
-
- @Test
- public void dummyTest() {
- Assert.assertTrue(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
deleted file mode 100644
index 5d08647..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-/**
- * This class tests the behavior of SCMDatanodeHeartbeatDispatcher.
- */
-public class TestSCMDatanodeHeartbeatDispatcher {
-
- @Test
- public void testSCMDatanodeHeartbeatDispatcherBuilder() {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeHeartbeatDispatcher dispatcher =
- SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
- .addHandlerFor(NodeReportProto.class)
- .addHandlerFor(ContainerReportsProto.class)
- .build();
- Assert.assertNotNull(dispatcher);
- }
-
- @Test
- public void testNodeReportDispatcher() throws IOException {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeNodeReportHandler nodeReportHandler =
- Mockito.mock(SCMDatanodeNodeReportHandler.class);
- SCMDatanodeHeartbeatDispatcher dispatcher =
- SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
- .addHandler(NodeReportProto.class, nodeReportHandler)
- .build();
-
- DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
- NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
- SCMHeartbeatRequestProto heartbeat =
- SCMHeartbeatRequestProto.newBuilder()
- .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
- .setNodeReport(nodeReport)
- .build();
- dispatcher.dispatch(heartbeat);
- verify(nodeReportHandler,
- times(1))
- .processReport(any(DatanodeDetails.class), eq(nodeReport));
- }
-
- @Test
- public void testContainerReportDispatcher() throws IOException {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeContainerReportHandler containerReportHandler =
- Mockito.mock(SCMDatanodeContainerReportHandler.class);
- SCMDatanodeHeartbeatDispatcher dispatcher =
- SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
- .addHandler(ContainerReportsProto.class, containerReportHandler)
- .build();
-
- DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
- ContainerReportsProto containerReport =
- ContainerReportsProto.getDefaultInstance();
- SCMHeartbeatRequestProto heartbeat =
- SCMHeartbeatRequestProto.newBuilder()
- .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
- .setContainerReport(containerReport)
- .build();
- dispatcher.dispatch(heartbeat);
- verify(containerReportHandler,
- times(1))
- .processReport(any(DatanodeDetails.class),
- any(ContainerReportsProto.class));
- }
-
- @Test
- public void testNodeAndContainerReportDispatcher() throws IOException {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeNodeReportHandler nodeReportHandler =
- Mockito.mock(SCMDatanodeNodeReportHandler.class);
- SCMDatanodeContainerReportHandler containerReportHandler =
- Mockito.mock(SCMDatanodeContainerReportHandler.class);
- SCMDatanodeHeartbeatDispatcher dispatcher =
- SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null)
- .addHandler(NodeReportProto.class, nodeReportHandler)
- .addHandler(ContainerReportsProto.class, containerReportHandler)
- .build();
-
- DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
- NodeReportProto nodeReport = NodeReportProto.getDefaultInstance();
- ContainerReportsProto containerReport =
- ContainerReportsProto.getDefaultInstance();
- SCMHeartbeatRequestProto heartbeat =
- SCMHeartbeatRequestProto.newBuilder()
- .setDatanodeDetails(datanodeDetails.getProtoBufMessage())
- .setNodeReport(nodeReport)
- .setContainerReport(containerReport)
- .build();
- dispatcher.dispatch(heartbeat);
- verify(nodeReportHandler,
- times(1))
- .processReport(any(DatanodeDetails.class), any(NodeReportProto.class));
- verify(containerReportHandler,
- times(1))
- .processReport(any(DatanodeDetails.class),
- any(ContainerReportsProto.class));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
deleted file mode 100644
index 30a753c..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify TestSCMDatanodeNodeReportHandler's behavior.
- */
-public class TestSCMDatanodeNodeReportHandler {
-
-
- //TODO: add test cases to verify SCMDatanodeNodeReportHandler.
-
- @Test
- public void dummyTest() {
- Assert.assertTrue(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
deleted file mode 100644
index 4b918f7..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.NodeReportProto;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test cases to verify the functionality of SCMDatanodeReportHandlerFactory.
- */
-public class TestSCMDatanodeReportHandlerFactory {
-
- @Test
- public void testNodeReportHandlerConstruction() {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeReportHandlerFactory factory =
- new SCMDatanodeReportHandlerFactory(conf, null);
- Assert.assertTrue(factory.getHandlerFor(NodeReportProto.class)
- instanceof SCMDatanodeNodeReportHandler);
- }
-
- @Test
- public void testContainerReporttHandlerConstruction() {
- Configuration conf = new OzoneConfiguration();
- SCMDatanodeReportHandlerFactory factory =
- new SCMDatanodeReportHandlerFactory(conf, null);
- Assert.assertTrue(factory.getHandlerFor(ContainerReportsProto.class)
- instanceof SCMDatanodeContainerReportHandler);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
deleted file mode 100644
index 4a3f59f..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.server.report;
-/**
- * Contains test-cases to test Datanode report handlers in SCM.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8752a485/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
deleted file mode 100644
index ecddf8e..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
-import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertEquals;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.server.report
- .SCMDatanodeContainerReportHandler;
-import org.apache.hadoop.metrics2.MetricsRecordBuilder;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerReport;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
-import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-/**
- * This class tests the metrics of Storage Container Manager.
- */
-public class TestSCMMetrics {
- /**
- * Set the timeout for each test.
- */
- @Rule
- public Timeout testTimeout = new Timeout(90000);
-
- private static MiniOzoneCluster cluster = null;
-
- @Test
- public void testContainerMetrics() throws Exception {
- int nodeCount = 2;
- int numReport = 2;
- long size = OzoneConsts.GB * 5;
- long used = OzoneConsts.GB * 2;
- long readBytes = OzoneConsts.GB * 1;
- long writeBytes = OzoneConsts.GB * 2;
- int keyCount = 1000;
- int readCount = 100;
- int writeCount = 50;
- OzoneConfiguration conf = new OzoneConfiguration();
-
- try {
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(nodeCount).build();
- cluster.waitForClusterToBeReady();
-
- ContainerStat stat = new ContainerStat(size, used, keyCount, readBytes,
- writeBytes, readCount, writeCount);
- StorageContainerManager scmManager = cluster.getStorageContainerManager();
- DatanodeDetails fstDatanodeDetails = TestUtils.getDatanodeDetails();
- ContainerReportsProto request = createContainerReport(numReport, stat);
- String fstDatanodeUuid = fstDatanodeDetails.getUuidString();
- SCMDatanodeContainerReportHandler containerReportHandler =
- new SCMDatanodeContainerReportHandler();
- containerReportHandler.setConf(conf);
- containerReportHandler.init(scmManager);
- containerReportHandler.processReport(
- fstDatanodeDetails, request);
-
- // verify container stat metrics
- MetricsRecordBuilder scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
- assertEquals(size * numReport,
- getLongGauge("LastContainerReportSize", scmMetrics));
- assertEquals(used * numReport,
- getLongGauge("LastContainerReportUsed", scmMetrics));
- assertEquals(readBytes * numReport,
- getLongGauge("LastContainerReportReadBytes", scmMetrics));
- assertEquals(writeBytes * numReport,
- getLongGauge("LastContainerReportWriteBytes", scmMetrics));
-
- assertEquals(keyCount * numReport,
- getLongGauge("LastContainerReportKeyCount", scmMetrics));
- assertEquals(readCount * numReport,
- getLongGauge("LastContainerReportReadCount", scmMetrics));
- assertEquals(writeCount * numReport,
- getLongGauge("LastContainerReportWriteCount", scmMetrics));
-
- // add one new report
- DatanodeDetails sndDatanodeDetails = TestUtils.getDatanodeDetails();
- request = createContainerReport(1, stat);
- String sndDatanodeUuid = sndDatanodeDetails.getUuidString();
- containerReportHandler.processReport(
- sndDatanodeDetails, request);
-
- scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
- assertEquals(size * (numReport + 1),
- getLongCounter("ContainerReportSize", scmMetrics));
- assertEquals(used * (numReport + 1),
- getLongCounter("ContainerReportUsed", scmMetrics));
- assertEquals(readBytes * (numReport + 1),
- getLongCounter("ContainerReportReadBytes", scmMetrics));
- assertEquals(writeBytes * (numReport + 1),
- getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
- assertEquals(keyCount * (numReport + 1),
- getLongCounter("ContainerReportKeyCount", scmMetrics));
- assertEquals(readCount * (numReport + 1),
- getLongCounter("ContainerReportReadCount", scmMetrics));
- assertEquals(writeCount * (numReport + 1),
- getLongCounter("ContainerReportWriteCount", scmMetrics));
-
- // Re-send reports but with different value for validating
- // the aggregation.
- stat = new ContainerStat(100, 50, 3, 50, 60, 5, 6);
- containerReportHandler.processReport(
- fstDatanodeDetails, createContainerReport(1, stat));
-
- stat = new ContainerStat(1, 1, 1, 1, 1, 1, 1);
- containerReportHandler.processReport(
- sndDatanodeDetails, createContainerReport(1, stat));
-
- // the global container metrics value should be updated
- scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
- assertEquals(101, getLongCounter("ContainerReportSize", scmMetrics));
- assertEquals(51, getLongCounter("ContainerReportUsed", scmMetrics));
- assertEquals(51, getLongCounter("ContainerReportReadBytes", scmMetrics));
- assertEquals(61, getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
- assertEquals(4, getLongCounter("ContainerReportKeyCount", scmMetrics));
- assertEquals(6, getLongCounter("ContainerReportReadCount", scmMetrics));
- assertEquals(7, getLongCounter("ContainerReportWriteCount", scmMetrics));
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- }
-
- @Test
- public void testStaleNodeContainerReport() throws Exception {
- int nodeCount = 2;
- int numReport = 2;
- long size = OzoneConsts.GB * 5;
- long used = OzoneConsts.GB * 2;
- long readBytes = OzoneConsts.GB * 1;
- long writeBytes = OzoneConsts.GB * 2;
- int keyCount = 1000;
- int readCount = 100;
- int writeCount = 50;
- OzoneConfiguration conf = new OzoneConfiguration();
-
- try {
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(nodeCount).build();
- cluster.waitForClusterToBeReady();
-
- ContainerStat stat = new ContainerStat(size, used, keyCount, readBytes,
- writeBytes, readCount, writeCount);
- StorageContainerManager scmManager = cluster.getStorageContainerManager();
-
- DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
- .getDatanodeDetails();
- SCMDatanodeContainerReportHandler containerReportHandler =
- new SCMDatanodeContainerReportHandler();
- containerReportHandler.setConf(conf);
- containerReportHandler.init(scmManager);
- ContainerReportsProto request = createContainerReport(numReport, stat);
- containerReportHandler.processReport(
- datanodeDetails, request);
-
- MetricsRecordBuilder scmMetrics = getMetrics(SCMMetrics.SOURCE_NAME);
- assertEquals(size * numReport,
- getLongCounter("ContainerReportSize", scmMetrics));
- assertEquals(used * numReport,
- getLongCounter("ContainerReportUsed", scmMetrics));
- assertEquals(readBytes * numReport,
- getLongCounter("ContainerReportReadBytes", scmMetrics));
- assertEquals(writeBytes * numReport,
- getLongCounter("ContainerReportWriteBytes", scmMetrics));
-
- assertEquals(keyCount * numReport,
- getLongCounter("ContainerReportKeyCount", scmMetrics));
- assertEquals(readCount * numReport,
- getLongCounter("ContainerReportReadCount", scmMetrics));
- assertEquals(writeCount * numReport,
- getLongCounter("ContainerReportWriteCount", scmMetrics));
-
- // reset stale interval time to move node from healthy to stale
- SCMNodeManager nodeManager = (SCMNodeManager) cluster
- .getStorageContainerManager().getScmNodeManager();
- nodeManager.setStaleNodeIntervalMs(100);
-
- // verify the metrics when node becomes stale
- GenericTestUtils.waitFor(() -> {
- MetricsRecordBuilder metrics = getMetrics(SCMMetrics.SOURCE_NAME);
- return 0 == getLongCounter("ContainerReportSize", metrics)
- && 0 == getLongCounter("ContainerReportUsed", metrics)
- && 0 == getLongCounter("ContainerReportReadBytes", metrics)
- && 0 == getLongCounter("ContainerReportWriteBytes", metrics)
- && 0 == getLongCounter("ContainerReportKeyCount", metrics)
- && 0 == getLongCounter("ContainerReportReadCount", metrics)
- && 0 == getLongCounter("ContainerReportWriteCount", metrics);
- }, 1000, 60000);
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
- }
-
- private ContainerReportsProto createContainerReport(int numReport,
- ContainerStat stat) {
- StorageContainerDatanodeProtocolProtos.ContainerReportsProto.Builder
- reportsBuilder = StorageContainerDatanodeProtocolProtos
- .ContainerReportsProto.newBuilder();
-
- for (int i = 0; i < numReport; i++) {
- ContainerReport report = new ContainerReport(
- RandomUtils.nextLong(), DigestUtils.sha256Hex("Simulated"));
- report.setSize(stat.getSize().get());
- report.setBytesUsed(stat.getUsed().get());
- report.setReadCount(stat.getReadCount().get());
- report.setReadBytes(stat.getReadBytes().get());
- report.setKeyCount(stat.getKeyCount().get());
- report.setWriteCount(stat.getWriteCount().get());
- report.setWriteBytes(stat.getWriteBytes().get());
- reportsBuilder.addReports(report.getProtoBufMessage());
- }
- return reportsBuilder.build();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-94. Change ozone datanode
command to start the standalone datanode plugin. Contributed by Sandeep
Nemuri.
Posted by gi...@apache.org.
HDDS-94. Change ozone datanode command to start the standalone datanode plugin.
Contributed by Sandeep Nemuri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18932717
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18932717
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18932717
Branch: refs/heads/HADOOP-15461
Commit: 18932717c42382ed8842de7719ec6d20c1765366
Parents: 1e30547
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Jun 26 18:28:47 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Jun 27 14:15:15 2018 -0700
----------------------------------------------------------------------
hadoop-dist/src/main/compose/ozone/docker-compose.yaml | 12 ------------
hadoop-dist/src/main/compose/ozone/docker-config | 5 -----
.../src/main/compose/ozoneperf/docker-compose.yaml | 13 -------------
hadoop-dist/src/main/compose/ozoneperf/docker-config | 5 -----
.../org/apache/hadoop/ozone/HddsDatanodeService.java | 13 ++++++++++++-
.../src/test/acceptance/basic/docker-compose.yaml | 12 ------------
.../src/test/acceptance/basic/docker-config | 5 -----
.../src/test/acceptance/commonlib.robot | 1 -
.../src/test/acceptance/ozonefs/docker-compose.yaml | 12 ------------
.../src/test/acceptance/ozonefs/docker-config | 5 -----
hadoop-ozone/common/src/main/bin/ozone | 9 +++------
11 files changed, 15 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
index faf420c..512c649 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
@@ -16,18 +16,6 @@
version: "3"
services:
- namenode:
- image: apache/hadoop-runner
- hostname: namenode
- volumes:
- - ../../ozone:/opt/hadoop
- ports:
- - 9870:9870
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
image: apache/hadoop-runner
volumes:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
index c693db0..632f870 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
index fb7873b..3233c11 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
@@ -16,19 +16,6 @@
version: "3"
services:
- namenode:
- image: apache/hadoop-runner
- hostname: namenode
- volumes:
- - ../../ozone:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9870:9870
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
image: apache/hadoop-runner
volumes:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config
index e4f5485..2be22a7 100644
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
@@ -23,12 +22,8 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index fa4187a..ddeec87 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,9 +25,11 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine;
+import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
@@ -229,9 +231,18 @@ public class HddsDatanodeService implements ServicePlugin {
public static void main(String[] args) {
try {
+ if (DFSUtil.parseHelpArgument(args, "Starts HDDS Datanode", System.out, false)) {
+ System.exit(0);
+ }
+ Configuration conf = new OzoneConfiguration();
+ GenericOptionsParser hParser = new GenericOptionsParser(conf, args);
+ if (!hParser.isParseSuccessful()) {
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ System.exit(1);
+ }
StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG);
HddsDatanodeService hddsDatanodeService =
- createHddsDatanodeService(new OzoneConfiguration());
+ createHddsDatanodeService(conf);
hddsDatanodeService.start(null);
hddsDatanodeService.join();
} catch (Throwable e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
index 44bd4a0..b50f42d 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
@@ -16,18 +16,6 @@
version: "3"
services:
- namenode:
- image: apache/hadoop-runner
- hostname: namenode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9870
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
image: apache/hadoop-runner
volumes:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
index 180dc8e..c3ec2ef 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
OZONE-SITE.XML_ozone.scm.names=scm
@@ -24,13 +23,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
index 01ed302..a5ea30a 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
@@ -28,7 +28,6 @@ Startup Ozone cluster with size
Daemons are running without error
Is daemon running without error ksm
Is daemon running without error scm
- Is daemon running without error namenode
Is daemon running without error datanode
Check if datanode is connected to the scm
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
index 3323557..12022df 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
@@ -16,18 +16,6 @@
version: "3"
services:
- namenode:
- image: apache/hadoop-runner
- hostname: namenode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9870
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
image: apache/hadoop-runner
volumes:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
index dec863e..e06d434 100644
--- a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
@@ -25,13 +24,9 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18932717/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 6843bdd..390f089 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -34,7 +34,7 @@ function hadoop_usage
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
- hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
+ hadoop_add_subcommand "datanode" daemon "run a HDDS datanode"
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
hadoop_add_subcommand "freon" client "runs an ozone data generator"
hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
@@ -45,7 +45,7 @@ function hadoop_usage
hadoop_add_subcommand "o3" client "command line interface for ozone"
hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
- hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
+ hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager"
hadoop_add_subcommand "version" client "print the version"
hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
@@ -68,10 +68,7 @@ function ozonecmd_case
;;
datanode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_SECURE_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
- HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
- hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR
- hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDDS-177. Create a releasable
ozonefs artifact Contributed by Marton, Elek.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
new file mode 100644
index 0000000..3156eb2
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract test for ROOT directory operations.
+ */
+public class ITestOzoneContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
new file mode 100644
index 0000000..c4bc0ff
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+
+/**
+ * Ozone contract tests covering file seek.
+ */
+public class ITestOzoneContractSeek extends AbstractContractSeekTest {
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster();
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
new file mode 100644
index 0000000..8417e46
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.junit.Assert;
+
+import java.io.IOException;
+
+/**
+ * The contract of Ozone: only enabled if the test bucket is provided.
+ */
+class OzoneContract extends AbstractFSContract {
+
+ private static MiniOzoneCluster cluster;
+ private static StorageHandler storageHandler;
+ private static final String CONTRACT_XML = "contract/ozone.xml";
+
+ OzoneContract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return OzoneConsts.OZONE_URI_SCHEME;
+ }
+
+ @Override
+ public Path getTestPath() {
+ Path path = new Path("/test");
+ return path;
+ }
+
+ public static void createCluster() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.addResource(CONTRACT_XML);
+
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+ try {
+ cluster.waitForClusterToBeReady();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
+ }
+
+ private void copyClusterConfigs(String configKey) {
+ getConf().set(configKey, cluster.getConf().get(configKey));
+ }
+
+ @Override
+ public FileSystem getTestFileSystem() throws IOException {
+ //assumes cluster is not null
+ Assert.assertNotNull("cluster not created", cluster);
+
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+
+
+ UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
+ volumeArgs.setUserName(userName);
+ volumeArgs.setAdminName(adminName);
+ BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
+ try {
+ storageHandler.createVolume(volumeArgs);
+ storageHandler.createBucket(bucketArgs);
+ } catch (OzoneException e) {
+ throw new IOException(e.getMessage());
+ }
+
+ String uri = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
+ getConf().set("fs.defaultFS", uri);
+ copyClusterConfigs(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY);
+ copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
+ return FileSystem.get(getConf());
+ }
+
+ public static void destroyCluster() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
new file mode 100644
index 0000000..fe2075c
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/contract/ozone.xml
@@ -0,0 +1,113 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ Ozone is a blobstore, with very different behavior than a classic filesystem.
+ -->
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.create-visibility-delayed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
new file mode 100644
index 0000000..3bf1619
--- /dev/null
+++ b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+
+# for debugging low level Ozone operations, uncomment this line
+# log4j.logger.org.apache.hadoop.ozone=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index cffef14..b655088 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -34,6 +34,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>client</module>
<module>ozone-manager</module>
<module>tools</module>
+ <module>ozonefs</module>
<module>integration-test</module>
<module>objectstore-service</module>
<module>docs</module>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ed0187b..dfd1eac 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -586,7 +586,11 @@
<artifactId>hadoop-ozone-docs</artifactId>
<version>${hdds.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/pom.xml b/hadoop-tools/hadoop-ozone/pom.xml
deleted file mode 100644
index a7d0cfa..0000000
--- a/hadoop-tools/hadoop-ozone/pom.xml
+++ /dev/null
@@ -1,174 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project</artifactId>
- <version>3.2.0-SNAPSHOT</version>
- <relativePath>../../hadoop-project</relativePath>
- </parent>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <name>Apache Hadoop Ozone FileSystem</name>
- <packaging>jar</packaging>
-
- <properties>
- <file.encoding>UTF-8</file.encoding>
- <downloadSources>true</downloadSources>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <executions>
- <execution>
- <id>deplist</id>
- <phase>compile</phase>
- <goals>
- <goal>list</goal>
- </goals>
- <configuration>
- <!-- build a shellprofile -->
- <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-scm</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-framework</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-ozone-manager</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-container-service</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-objectstore-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-integration-test</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-distcp</artifactId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
deleted file mode 100644
index 832a0cb..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-/**
- * Constants for Ozone FileSystem implementation.
- */
-public final class Constants {
-
- public static final String OZONE_DEFAULT_USER = "hdfs";
-
- public static final String OZONE_USER_DIR = "/user";
-
- /** Local buffer directory. */
- public static final String BUFFER_DIR_KEY = "fs.ozone.buffer.dir";
-
- /** Temporary directory. */
- public static final String BUFFER_TMP_KEY = "hadoop.tmp.dir";
-
- /** Page size for Ozone listing operation. */
- public static final int LISTING_PAGE_SIZE = 1024;
-
- private Constants() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
deleted file mode 100644
index 4163c13..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * ozone implementation of AbstractFileSystem.
- * This impl delegates to the OzoneFileSystem
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class OzFs extends DelegateToFileSystem {
-
- public OzFs(URI theUri, Configuration conf)
- throws IOException, URISyntaxException {
- super(theUri, new OzoneFileSystem(), conf,
- OzoneConsts.OZONE_URI_SCHEME, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
deleted file mode 100644
index 4c5c0c8..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSInputStream.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * The input stream for Ozone file system.
- *
- * TODO: Make inputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class OzoneFSInputStream extends FSInputStream {
-
- private final ChunkGroupInputStream inputStream;
-
- public OzoneFSInputStream(InputStream inputStream) {
- this.inputStream = (ChunkGroupInputStream)inputStream;
- }
-
- @Override
- public int read() throws IOException {
- return inputStream.read();
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return inputStream.read(b, off, len);
- }
-
- @Override
- public synchronized void close() throws IOException {
- inputStream.close();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- inputStream.seek(pos);
- }
-
- @Override
- public long getPos() throws IOException {
- return inputStream.getPos();
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- @Override
- public int available() throws IOException {
- return inputStream.available();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
deleted file mode 100644
index faa3628..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFSOutputStream.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-
-
-/**
- * The output stream for Ozone file system.
- *
- * TODO: Make outputStream generic for both rest and rpc clients
- * This class is not thread safe.
- */
-public class OzoneFSOutputStream extends OutputStream {
-
- private final ChunkGroupOutputStream outputStream;
-
- public OzoneFSOutputStream(OutputStream outputStream) {
- this.outputStream = (ChunkGroupOutputStream)outputStream;
- }
-
- @Override
- public void write(int b) throws IOException {
- outputStream.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- outputStream.write(b, off, len);
- }
-
- @Override
- public synchronized void flush() throws IOException {
- outputStream.flush();
- }
-
- @Override
- public synchronized void close() throws IOException {
- outputStream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
deleted file mode 100644
index 6906a9d..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Iterator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.ObjectStore;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
-import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
-
-/**
- * The Ozone Filesystem implementation.
- *
- * This subclass is marked as private as code should not be creating it
- * directly; use {@link FileSystem#get(Configuration)} and variants to create
- * one. If cast to {@link OzoneFileSystem}, extra methods and features may be
- * accessed. Consider those private and unstable.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class OzoneFileSystem extends FileSystem {
- static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class);
-
- /** The Ozone client for connecting to Ozone server. */
- private OzoneClient ozoneClient;
- private ObjectStore objectStore;
- private OzoneVolume volume;
- private OzoneBucket bucket;
- private URI uri;
- private String userName;
- private Path workingDir;
- private ReplicationType replicationType;
- private ReplicationFactor replicationFactor;
-
- private static final Pattern URL_SCHEMA_PATTERN =
- Pattern.compile("(.+)\\.([^\\.]+)");
-
- @Override
- public void initialize(URI name, Configuration conf) throws IOException {
- super.initialize(name, conf);
- setConf(conf);
- Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name);
- assert getScheme().equals(name.getScheme());
-
- String authority = name.getAuthority();
-
- Matcher matcher = URL_SCHEMA_PATTERN.matcher(authority);
-
- if (!matcher.matches()) {
- throw new IllegalArgumentException("Ozone file system url should be "
- + "in the form o3://bucket.volume");
- }
- String bucketStr = matcher.group(1);
- String volumeStr = matcher.group(2);
-
- try {
- uri = new URIBuilder().setScheme(OZONE_URI_SCHEME)
- .setHost(authority).build();
- LOG.trace("Ozone URI for ozfs initialization is " + uri);
- this.ozoneClient = OzoneClientFactory.getRpcClient(conf);
- objectStore = ozoneClient.getObjectStore();
- this.volume = objectStore.getVolume(volumeStr);
- this.bucket = volume.getBucket(bucketStr);
- this.replicationType = ReplicationType.valueOf(
- conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
- OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
- this.replicationFactor = ReplicationFactor.valueOf(
- conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
- OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
- try {
- this.userName =
- UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- this.userName = OZONE_DEFAULT_USER;
- }
- this.workingDir = new Path(OZONE_USER_DIR, this.userName)
- .makeQualified(this.uri, this.workingDir);
- } catch (URISyntaxException ue) {
- final String msg = "Invalid Ozone endpoint " + name;
- LOG.error(msg, ue);
- throw new IOException(msg, ue);
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- ozoneClient.close();
- } finally {
- super.close();
- }
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
-
- @Override
- public String getScheme() {
- return OZONE_URI_SCHEME;
- }
-
- @Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- LOG.trace("open() path:{}", f);
- final FileStatus fileStatus = getFileStatus(f);
- final String key = pathToKey(f);
- if (fileStatus.isDirectory()) {
- throw new FileNotFoundException("Can't open directory " + f + " to read");
- }
-
- return new FSDataInputStream(
- new OzoneFSInputStream(bucket.readKey(key).getInputStream()));
- }
-
- @Override
- public FSDataOutputStream create(Path f, FsPermission permission,
- boolean overwrite, int bufferSize,
- short replication, long blockSize,
- Progressable progress) throws IOException {
- LOG.trace("create() path:{}", f);
- final String key = pathToKey(f);
- final FileStatus status;
- try {
- status = getFileStatus(f);
- if (status.isDirectory()) {
- throw new FileAlreadyExistsException(f + " is a directory");
- } else {
- if (!overwrite) {
- // path references a file and overwrite is disabled
- throw new FileAlreadyExistsException(f + " already exists");
- }
- LOG.trace("Overwriting file {}", f);
- deleteObject(key);
- }
- } catch (FileNotFoundException ignored) {
- // check if the parent directory needs to be created
- Path parent = f.getParent();
- try {
- // create all the directories for the parent
- FileStatus parentStatus = getFileStatus(parent);
- LOG.trace("parent key:{} status:{}", key, parentStatus);
- } catch (FileNotFoundException e) {
- mkdirs(parent);
- }
- // This exception needs to ignored as this means that the file currently
- // does not exists and a new file can thus be created.
- }
-
- OzoneOutputStream ozoneOutputStream =
- bucket.createKey(key, 0, replicationType, replicationFactor);
- // We pass null to FSDataOutputStream so it won't count writes that
- // are being buffered to a file
- return new FSDataOutputStream(
- new OzoneFSOutputStream(ozoneOutputStream.getOutputStream()), null);
- }
-
- @Override
- public FSDataOutputStream createNonRecursive(Path path,
- FsPermission permission,
- EnumSet<CreateFlag> flags,
- int bufferSize,
- short replication,
- long blockSize,
- Progressable progress) throws IOException {
- final Path parent = path.getParent();
- if (parent != null) {
- // expect this to raise an exception if there is no parent
- if (!getFileStatus(parent).isDirectory()) {
- throw new FileAlreadyExistsException("Not a directory: " + parent);
- }
- }
- return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
- bufferSize, replication, blockSize, progress);
- }
-
- @Override
- public FSDataOutputStream append(Path f, int bufferSize,
- Progressable progress) throws IOException {
- throw new UnsupportedOperationException("append() Not implemented by the "
- + getClass().getSimpleName() + " FileSystem implementation");
- }
-
- private class RenameIterator extends OzoneListingIterator {
- private final String srcKey;
- private final String dstKey;
-
- RenameIterator(Path srcPath, Path dstPath)
- throws IOException {
- super(srcPath);
- srcKey = pathToKey(srcPath);
- dstKey = pathToKey(dstPath);
- LOG.trace("rename from:{} to:{}", srcKey, dstKey);
- }
-
- boolean processKey(String key) throws IOException {
- String newKeyName = dstKey.concat(key.substring(srcKey.length()));
- bucket.renameKey(key, newKeyName);
- return true;
- }
- }
-
- /**
- * Check whether the source and destination path are valid and then perform
- * rename from source path to destination path.
- *
- * The rename operation is performed by renaming the keys with src as prefix.
- * For such keys the prefix is changed from src to dst.
- *
- * @param src source path for rename
- * @param dst destination path for rename
- * @return true if rename operation succeeded or
- * if the src and dst have the same path and are of the same type
- * @throws IOException on I/O errors or if the src/dst paths are invalid.
- */
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- if (src.equals(dst)) {
- return true;
- }
-
- LOG.trace("rename() from:{} to:{}", src, dst);
- if (src.isRoot()) {
- // Cannot rename root of file system
- LOG.trace("Cannot rename the root of a filesystem");
- return false;
- }
-
- // Cannot rename a directory to its own subdirectory
- Path dstParent = dst.getParent();
- while (dstParent != null && !src.equals(dstParent)) {
- dstParent = dstParent.getParent();
- }
- Preconditions.checkArgument(dstParent == null,
- "Cannot rename a directory to its own subdirectory");
- // Check if the source exists
- FileStatus srcStatus;
- try {
- srcStatus = getFileStatus(src);
- } catch (FileNotFoundException fnfe) {
- // source doesn't exist, return
- return false;
- }
-
- // Check if the destination exists
- FileStatus dstStatus;
- try {
- dstStatus = getFileStatus(dst);
- } catch (FileNotFoundException fnde) {
- dstStatus = null;
- }
-
- if (dstStatus == null) {
- // If dst doesn't exist, check whether dst parent dir exists or not
- // if the parent exists, the source can still be renamed to dst path
- dstStatus = getFileStatus(dst.getParent());
- if (!dstStatus.isDirectory()) {
- throw new IOException(String.format(
- "Failed to rename %s to %s, %s is a file", src, dst,
- dst.getParent()));
- }
- } else {
- // if dst exists and source and destination are same,
- // check both the src and dst are of same type
- if (srcStatus.getPath().equals(dstStatus.getPath())) {
- return !srcStatus.isDirectory();
- } else if (dstStatus.isDirectory()) {
- // If dst is a directory, rename source as subpath of it.
- // for example rename /source to /dst will lead to /dst/source
- dst = new Path(dst, src.getName());
- FileStatus[] statuses;
- try {
- statuses = listStatus(dst);
- } catch (FileNotFoundException fnde) {
- statuses = null;
- }
-
- if (statuses != null && statuses.length > 0) {
- // If dst exists and not a directory not empty
- throw new FileAlreadyExistsException(String.format(
- "Failed to rename %s to %s, file already exists or not empty!",
- src, dst));
- }
- } else {
- // If dst is not a directory
- throw new FileAlreadyExistsException(String.format(
- "Failed to rename %s to %s, file already exists!", src, dst));
- }
- }
-
- if (srcStatus.isDirectory()) {
- if (dst.toString().startsWith(src.toString())) {
- LOG.trace("Cannot rename a directory to a subdirectory of self");
- return false;
- }
- }
- RenameIterator iterator = new RenameIterator(src, dst);
- return iterator.iterate();
- }
-
- private class DeleteIterator extends OzoneListingIterator {
- private boolean recursive;
- DeleteIterator(Path f, boolean recursive)
- throws IOException {
- super(f);
- this.recursive = recursive;
- if (getStatus().isDirectory()
- && !this.recursive
- && listStatus(f).length != 0) {
- throw new PathIsNotEmptyDirectoryException(f.toString());
- }
- }
-
- boolean processKey(String key) throws IOException {
- if (key.equals("")) {
- LOG.trace("Skipping deleting root directory");
- return true;
- } else {
- LOG.trace("deleting key:" + key);
- boolean succeed = deleteObject(key);
- // if recursive delete is requested ignore the return value of
- // deleteObject and issue deletes for other keys.
- return recursive || succeed;
- }
- }
- }
-
- @Override
- public boolean delete(Path f, boolean recursive) throws IOException {
- LOG.trace("delete() path:{} recursive:{}", f, recursive);
- try {
- DeleteIterator iterator = new DeleteIterator(f, recursive);
- return iterator.iterate();
- } catch (FileNotFoundException e) {
- LOG.debug("Couldn't delete {} - does not exist", f);
- return false;
- }
- }
-
- private class ListStatusIterator extends OzoneListingIterator {
- private List<FileStatus> statuses = new ArrayList<>(LISTING_PAGE_SIZE);
- private Path f;
-
- ListStatusIterator(Path f) throws IOException {
- super(f);
- this.f = f;
- }
-
- boolean processKey(String key) throws IOException {
- Path keyPath = new Path(OZONE_URI_DELIMITER + key);
- if (key.equals(getPathKey())) {
- if (pathIsDirectory()) {
- return true;
- } else {
- statuses.add(getFileStatus(keyPath));
- return true;
- }
- }
- // left with only subkeys now
- if (pathToKey(keyPath.getParent()).equals(pathToKey(f))) {
- // skip keys which are for subdirectories of the directory
- statuses.add(getFileStatus(keyPath));
- }
- return true;
- }
-
- FileStatus[] getStatuses() {
- return statuses.toArray(new FileStatus[statuses.size()]);
- }
- }
-
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
- LOG.trace("listStatus() path:{}", f);
- ListStatusIterator iterator = new ListStatusIterator(f);
- iterator.iterate();
- return iterator.getStatuses();
- }
-
- @Override
- public void setWorkingDirectory(Path newDir) {
- workingDir = newDir;
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- /**
- * Check whether the path is valid and then create directories.
- * Directory is represented using a key with no value.
- * All the non-existent parent directories are also created.
- *
- * @param path directory path to be created
- * @return true if directory exists or created successfully.
- * @throws IOException
- */
- private boolean mkdir(Path path) throws IOException {
- Path fPart = path;
- Path prevfPart = null;
- do {
- LOG.trace("validating path:{}", fPart);
- try {
- FileStatus fileStatus = getFileStatus(fPart);
- if (fileStatus.isDirectory()) {
- // If path exists and a directory, exit
- break;
- } else {
- // Found a file here, rollback and delete newly created directories
- LOG.trace("Found a file with same name as directory, path:{}", fPart);
- if (prevfPart != null) {
- delete(prevfPart, true);
- }
- throw new FileAlreadyExistsException(String.format(
- "Can't make directory for path '%s', it is a file.", fPart));
- }
- } catch (FileNotFoundException fnfe) {
- LOG.trace("creating directory for fpart:{}", fPart);
- String key = pathToKey(fPart);
- String dirKey = addTrailingSlashIfNeeded(key);
- if (!createDirectory(dirKey)) {
- // Directory creation failed here,
- // rollback and delete newly created directories
- LOG.trace("Directory creation failed, path:{}", fPart);
- if (prevfPart != null) {
- delete(prevfPart, true);
- }
- return false;
- }
- }
- prevfPart = fPart;
- fPart = fPart.getParent();
- } while (fPart != null);
- return true;
- }
-
- @Override
- public boolean mkdirs(Path f, FsPermission permission) throws IOException {
- LOG.trace("mkdir() path:{} ", f);
- String key = pathToKey(f);
- if (StringUtils.isEmpty(key)) {
- return false;
- }
- return mkdir(f);
- }
-
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- LOG.trace("getFileStatus() path:{}", f);
- Path qualifiedPath = f.makeQualified(uri, workingDir);
- String key = pathToKey(qualifiedPath);
-
- if (key.length() == 0) {
- return new FileStatus(0, true, 1, 0,
- bucket.getCreationTime(), qualifiedPath);
- }
-
- // consider this a file and get key status
- OzoneKey meta = getKeyInfo(key);
- if (meta == null) {
- key = addTrailingSlashIfNeeded(key);
- meta = getKeyInfo(key);
- }
-
- if (meta == null) {
- LOG.trace("File:{} not found", f);
- throw new FileNotFoundException(f + ": No such file or directory!");
- } else if (isDirectory(meta)) {
- return new FileStatus(0, true, 1, 0,
- meta.getModificationTime(), qualifiedPath);
- } else {
- //TODO: Fetch replication count from ratis config
- return new FileStatus(meta.getDataSize(), false, 1,
- getDefaultBlockSize(f), meta.getModificationTime(), qualifiedPath);
- }
- }
-
- /**
- * Helper method to fetch the key metadata info.
- * @param key key whose metadata information needs to be fetched
- * @return metadata info of the key
- */
- private OzoneKey getKeyInfo(String key) {
- try {
- return bucket.getKey(key);
- } catch (IOException e) {
- LOG.trace("Key:{} does not exists", key);
- return null;
- }
- }
-
- /**
- * Helper method to check if an Ozone key is representing a directory.
- * @param key key to be checked as a directory
- * @return true if key is a directory, false otherwise
- */
- private boolean isDirectory(OzoneKey key) {
- LOG.trace("key name:{} size:{}", key.getName(),
- key.getDataSize());
- return key.getName().endsWith(OZONE_URI_DELIMITER)
- && (key.getDataSize() == 0);
- }
-
- /**
- * Helper method to create an directory specified by key name in bucket.
- * @param keyName key name to be created as directory
- * @return true if the key is created, false otherwise
- */
- private boolean createDirectory(String keyName) {
- try {
- LOG.trace("creating dir for key:{}", keyName);
- bucket.createKey(keyName, 0, replicationType, replicationFactor).close();
- return true;
- } catch (IOException ioe) {
- LOG.error("create key failed for key:{}", keyName, ioe);
- return false;
- }
- }
-
- /**
- * Helper method to delete an object specified by key name in bucket.
- * @param keyName key name to be deleted
- * @return true if the key is deleted, false otherwise
- */
- private boolean deleteObject(String keyName) {
- LOG.trace("issuing delete for key" + keyName);
- try {
- bucket.deleteKey(keyName);
- return true;
- } catch (IOException ioe) {
- LOG.error("delete key failed " + ioe.getMessage());
- return false;
- }
- }
-
- /**
- * Turn a path (relative or otherwise) into an Ozone key.
- *
- * @param path the path of the file.
- * @return the key of the object that represents the file.
- */
- public String pathToKey(Path path) {
- Objects.requireNonNull(path, "Path can not be null!");
- if (!path.isAbsolute()) {
- path = new Path(workingDir, path);
- }
- // removing leading '/' char
- String key = path.toUri().getPath().substring(1);
- LOG.trace("path for key:{} is:{}", key, path);
- return key;
- }
-
- /**
- * Add trailing delimiter to path if it is already not present.
- *
- * @param key the ozone Key which needs to be appended
- * @return delimiter appended key
- */
- private String addTrailingSlashIfNeeded(String key) {
- if (StringUtils.isNotEmpty(key) && !key.endsWith(OZONE_URI_DELIMITER)) {
- return key + OZONE_URI_DELIMITER;
- } else {
- return key;
- }
- }
-
- @Override
- public String toString() {
- return "OzoneFileSystem{URI=" + uri + ", "
- + "workingDir=" + workingDir + ", "
- + "userName=" + userName + ", "
- + "statistics=" + statistics
- + "}";
- }
-
- private abstract class OzoneListingIterator {
- private final Path path;
- private final FileStatus status;
- private String pathKey;
- private Iterator<OzoneKey> keyIterator;
-
- OzoneListingIterator(Path path)
- throws IOException {
- this.path = path;
- this.status = getFileStatus(path);
- this.pathKey = pathToKey(path);
- if (status.isDirectory()) {
- this.pathKey = addTrailingSlashIfNeeded(pathKey);
- }
- keyIterator = bucket.listKeys(pathKey);
- }
-
- abstract boolean processKey(String key) throws IOException;
-
- // iterates all the keys in the particular path
- boolean iterate() throws IOException {
- LOG.trace("Iterating path {}", path);
- if (status.isDirectory()) {
- LOG.trace("Iterating directory:{}", pathKey);
- while (keyIterator.hasNext()) {
- OzoneKey key = keyIterator.next();
- LOG.trace("iterating key:{}", key.getName());
- if (!processKey(key.getName())) {
- return false;
- }
- }
- return true;
- } else {
- LOG.trace("iterating file:{}", path);
- return processKey(pathKey);
- }
- }
-
- String getPathKey() {
- return pathKey;
- }
-
- boolean pathIsDirectory() {
- return status.isDirectory();
- }
-
- FileStatus getStatus() {
- return status;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
deleted file mode 100644
index 93e82c3..0000000
--- a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-/**
- * Ozone Filesystem.
- *
- * Except for the exceptions, it should all be hidden as implementation details.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
deleted file mode 100644
index ad21f28..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * Test OzoneFSInputStream by reading through multiple interfaces.
- */
-public class TestOzoneFSInputStream {
- private static MiniOzoneCluster cluster = null;
- private static FileSystem fs;
- private static StorageHandler storageHandler;
- private static Path filePath = null;
- private static byte[] data = null;
-
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true and
- * OZONE_HANDLER_TYPE_KEY = "distributed"
- *
- * @throws IOException
- */
- @BeforeClass
- public static void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(10)
- .build();
- cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
-
- // create a volume and a bucket to be used by OzoneFileSystem
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- // Fetch the host and port for File System init
- DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
- .getDatanodeDetails();
- int port = datanodeDetails
- .getPort(DatanodeDetails.Port.Name.REST).getValue();
- String host = datanodeDetails.getHostName();
-
- // Set the fs.defaultFS and start the filesystem
- String uri = String.format("%s://%s.%s/",
- OzoneConsts.OZONE_URI_SCHEME, bucketName, volumeName);
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri);
- fs = FileSystem.get(conf);
- int fileLen = 100 * 1024 * 1024;
- data = DFSUtil.string2Bytes(RandomStringUtils.randomAlphanumeric(fileLen));
- filePath = new Path("/" + RandomStringUtils.randomAlphanumeric(5));
- try (FSDataOutputStream stream = fs.create(filePath)) {
- stream.write(data);
- }
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() throws IOException {
- fs.close();
- storageHandler.close();
- cluster.shutdown();
- }
-
- @Test
- public void testO3FSSingleByteRead() throws IOException {
- FSDataInputStream inputStream = fs.open(filePath);
- byte[] value = new byte[data.length];
- int i = 0;
- while(true) {
- int val = inputStream.read();
- if (val == -1) {
- break;
- }
- value[i] = (byte)val;
- Assert.assertEquals("value mismatch at:" + i, value[i], data[i]);
- i++;
- }
- Assert.assertEquals(i, data.length);
- Assert.assertTrue(Arrays.equals(value, data));
- inputStream.close();
- }
-
- @Test
- public void testO3FSMultiByteRead() throws IOException {
- FSDataInputStream inputStream = fs.open(filePath);
- byte[] value = new byte[data.length];
- byte[] tmp = new byte[1* 1024 *1024];
- int i = 0;
- while(true) {
- int val = inputStream.read(tmp);
- if (val == -1) {
- break;
- }
- System.arraycopy(tmp, 0, value, i * tmp.length, tmp.length);
- i++;
- }
- Assert.assertEquals(i * tmp.length, data.length);
- Assert.assertTrue(Arrays.equals(value, data));
- inputStream.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
deleted file mode 100644
index a225702..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.junit.After;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-
-import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test OzoneFileSystem Interfaces.
- *
- * This test will test the various interfaces i.e.
- * create, read, write, getFileStatus
- */
-@RunWith(Parameterized.class)
-public class TestOzoneFileInterfaces {
-
- private String rootPath;
- private String userName;
-
- /**
- * Parameter class to set absolute url/defaultFS handling.
- * <p>
- * Hadoop file systems could be used in multiple ways: Using the defaultfs
- * and file path without the schema, or use absolute url-s even with
- * different defaultFS. This parameter matrix would test both the use cases.
- */
- @Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {{false, true}, {true, false}});
- }
-
- private boolean setDefaultFs;
-
- private boolean useAbsolutePath;
-
- private static MiniOzoneCluster cluster = null;
-
- private static FileSystem fs;
-
- private static StorageHandler storageHandler;
-
- public TestOzoneFileInterfaces(boolean setDefaultFs,
- boolean useAbsolutePath) {
- this.setDefaultFs = setDefaultFs;
- this.useAbsolutePath = useAbsolutePath;
- }
-
- @Before
- public void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(3)
- .build();
- cluster.waitForClusterToBeReady();
- storageHandler =
- new ObjectStoreHandler(conf).getStorageHandler();
-
- // create a volume and a bucket to be used by OzoneFileSystem
- userName = "user" + RandomStringUtils.randomNumeric(5);
- String adminName = "admin" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
- UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
- volumeArgs.setUserName(userName);
- volumeArgs.setAdminName(adminName);
- storageHandler.createVolume(volumeArgs);
- BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
- storageHandler.createBucket(bucketArgs);
-
- rootPath = String
- .format("%s://%s.%s/", OzoneConsts.OZONE_URI_SCHEME, bucketName,
- volumeName);
- if (setDefaultFs) {
- // Set the fs.defaultFS and start the filesystem
- conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
- fs = FileSystem.get(conf);
- } else {
- fs = FileSystem.get(new URI(rootPath + "/test.txt"), conf);
- }
- }
-
- @After
- public void teardown() throws IOException {
- if (cluster != null) {
- cluster.shutdown();
- }
- IOUtils.closeQuietly(fs);
- IOUtils.closeQuietly(storageHandler);
- }
-
- @Test
- public void testFileSystemInit() throws IOException {
- if (setDefaultFs) {
- assertTrue(
- "The initialized file system is not OzoneFileSystem but " +
- fs.getClass(),
- fs instanceof OzoneFileSystem);
- assertEquals(OzoneConsts.OZONE_URI_SCHEME, fs.getUri().getScheme());
- }
- }
-
- @Test
- public void testOzFsReadWrite() throws IOException {
- long currentTime = Time.now();
- int stringLen = 20;
- String data = RandomStringUtils.randomAlphanumeric(stringLen);
- String filePath = RandomStringUtils.randomAlphanumeric(5);
- Path path = createPath("/" + filePath);
- try (FSDataOutputStream stream = fs.create(path)) {
- stream.writeBytes(data);
- }
-
- FileStatus status = fs.getFileStatus(path);
- // The timestamp of the newly created file should always be greater than
- // the time when the test was started
- assertTrue("Modification time has not been recorded: " + status,
- status.getModificationTime() > currentTime);
-
- try (FSDataInputStream inputStream = fs.open(path)) {
- byte[] buffer = new byte[stringLen];
- inputStream.readFully(0, buffer);
- String out = new String(buffer, 0, buffer.length);
- assertEquals(data, out);
- }
- }
-
-
- @Test
- public void testDirectory() throws IOException {
- String dirPath = RandomStringUtils.randomAlphanumeric(5);
- Path path = createPath("/" + dirPath);
- assertTrue("Makedirs returned with false for the path " + path,
- fs.mkdirs(path));
-
- FileStatus status = fs.getFileStatus(path);
- assertTrue("The created path is not directory.", status.isDirectory());
-
- assertEquals(0, status.getLen());
-
- FileStatus[] statusList = fs.listStatus(createPath("/"));
- assertEquals(1, statusList.length);
- assertEquals(status, statusList[0]);
-
- FileStatus statusRoot = fs.getFileStatus(createPath("/"));
- assertTrue("Root dir (/) is not a directory.", status.isDirectory());
- assertEquals(0, status.getLen());
-
-
- }
-
- @Test
- public void testPathToKey() throws Exception {
- OzoneFileSystem ozoneFs = (OzoneFileSystem) TestOzoneFileInterfaces.fs;
-
- assertEquals("a/b/1", ozoneFs.pathToKey(new Path("/a/b/1")));
-
- assertEquals("user/" + getCurrentUser() + "/key1/key2",
- ozoneFs.pathToKey(new Path("key1/key2")));
-
- assertEquals("key1/key2",
- ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
- }
-
- private String getCurrentUser() {
- try {
- return UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- return OZONE_DEFAULT_USER;
- }
- }
-
- private Path createPath(String relativePath) {
- if (useAbsolutePath) {
- return new Path(
- rootPath + (relativePath.startsWith("/") ? "" : "/") + relativePath);
- } else {
- return new Path(relativePath);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
deleted file mode 100644
index dd54315..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests creating files.
- */
-public class ITestOzoneContractCreate extends AbstractContractCreateTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
deleted file mode 100644
index f0a3d8d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering deletes.
- */
-public class ITestOzoneContractDelete extends AbstractContractDeleteTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
deleted file mode 100644
index 134a9ad..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-
-/**
- * Contract test suite covering S3A integration with DistCp.
- * Uses the block output stream, buffered to disk. This is the
- * recommended output mechanism for DistCP due to its scalability.
- */
-public class ITestOzoneContractDistCp extends AbstractContractDistCpTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected OzoneContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
deleted file mode 100644
index 98bbb14..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering getFileStatus.
- */
-public class ITestOzoneContractGetFileStatus
- extends AbstractContractGetFileStatusTest {
-
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
- @Override
- public void teardown() throws Exception {
- getLog().info("FS details {}", getFileSystem());
- super.teardown();
- }
-
- @Override
- protected Configuration createConfiguration() {
- return super.createConfiguration();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
deleted file mode 100644
index bc0de5d..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Test dir operations on Ozone.
- */
-public class ITestOzoneContractMkdir extends AbstractContractMkdirTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
deleted file mode 100644
index 0bc57d4..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests opening files.
- */
-public class ITestOzoneContractOpen extends AbstractContractOpenTest {
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e16e5b30/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
deleted file mode 100644
index 8ce1d1b..0000000
--- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.ozone.contract;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
-import org.apache.hadoop.fs.contract.AbstractFSContract;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-
-/**
- * Ozone contract tests covering rename.
- */
-public class ITestOzoneContractRename extends AbstractContractRenameTest {
-
- @BeforeClass
- public static void createCluster() throws IOException {
- OzoneContract.createCluster();
- }
-
- @AfterClass
- public static void teardownCluster() throws IOException {
- OzoneContract.destroyCluster();
- }
-
- @Override
- protected AbstractFSContract createContract(Configuration conf) {
- return new OzoneContract(conf);
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-15495. Upgrade commons-lang
version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by
Takanobu Asanuma.
Posted by gi...@apache.org.
HADOOP-15495. Upgrade commons-lang version to 3.7 in hadoop-common-project and hadoop-tools. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2399d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2399d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2399d6
Branch: refs/heads/HADOOP-15461
Commit: 2b2399d623539ab68e71a38fa9fbfc9a405bddb8
Parents: 8752a48
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Jun 28 14:29:40 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Jun 28 14:37:22 2018 +0900
----------------------------------------------------------------------
.../hadoop-client-minicluster/pom.xml | 8 ---
hadoop-common-project/hadoop-common/pom.xml | 5 --
.../hadoop/conf/ReconfigurationServlet.java | 42 ++++++-------
.../apache/hadoop/crypto/key/KeyProvider.java | 4 +-
.../org/apache/hadoop/crypto/key/KeyShell.java | 2 +-
.../main/java/org/apache/hadoop/fs/FsShell.java | 3 +-
.../main/java/org/apache/hadoop/fs/Path.java | 2 +-
.../java/org/apache/hadoop/fs/shell/Count.java | 2 +-
.../apache/hadoop/io/ElasticByteBufferPool.java | 2 +-
.../apache/hadoop/io/erasurecode/ECSchema.java | 4 +-
.../nativeio/SharedFileDescriptorFactory.java | 2 +-
.../org/apache/hadoop/ipc/CallerContext.java | 4 +-
.../apache/hadoop/ipc/DecayRpcScheduler.java | 4 +-
.../org/apache/hadoop/ipc/FairCallQueue.java | 4 +-
.../hadoop/metrics2/MetricsJsonBuilder.java | 2 +-
.../hadoop/metrics2/lib/MethodMetric.java | 2 +-
.../metrics2/lib/MutableMetricsFactory.java | 2 +-
.../hadoop/metrics2/lib/MutableQuantiles.java | 2 +-
.../metrics2/lib/MutableRollingAverages.java | 2 +-
.../apache/hadoop/metrics2/lib/MutableStat.java | 2 +-
.../metrics2/sink/RollingFileSystemSink.java | 2 +-
.../org/apache/hadoop/net/TableMapping.java | 2 +-
.../apache/hadoop/net/unix/DomainSocket.java | 2 +-
.../hadoop/net/unix/DomainSocketWatcher.java | 2 +-
.../security/ShellBasedUnixGroupsMapping.java | 2 +-
.../hadoop/security/alias/CredentialShell.java | 2 +-
.../hadoop/security/http/CrossOriginFilter.java | 2 +-
.../hadoop/security/token/DtFileOperations.java | 2 +-
.../org/apache/hadoop/tools/TableListing.java | 6 +-
.../org/apache/hadoop/util/StringUtils.java | 64 +++++++++++++++++++-
.../apache/hadoop/conf/TestConfiguration.java | 2 +-
.../conf/TestConfigurationFieldsBase.java | 2 +-
.../crypto/random/TestOsSecureRandom.java | 2 +-
.../org/apache/hadoop/fs/FSTestWrapper.java | 2 +-
.../apache/hadoop/fs/FileContextTestHelper.java | 1 -
.../hadoop/fs/TestDFCachingGetSpaceUsed.java | 2 +-
.../fs/TestFileSystemStorageStatistics.java | 18 +++---
.../hadoop/fs/shell/TestCopyFromLocal.java | 10 +--
.../TestSharedFileDescriptorFactory.java | 2 +-
.../org/apache/hadoop/ipc/TestProtoBufRpc.java | 2 +-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 2 +-
.../org/apache/hadoop/net/TestNetUtils.java | 2 +-
.../hadoop/net/unix/TestDomainSocket.java | 2 +-
...tionTokenAuthenticationHandlerWithMocks.java | 2 +-
.../AbstractServiceLauncherTestBase.java | 2 +-
.../apache/hadoop/test/GenericTestUtils.java | 4 +-
.../hadoop/util/TestShutdownHookManager.java | 2 +-
.../apache/hadoop/util/TestSignalLogger.java | 2 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 2 +-
hadoop-project/pom.xml | 5 --
.../aliyun/oss/AliyunCredentialsProvider.java | 2 +-
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 2 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 2 +-
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 2 +-
.../fs/aliyun/oss/AliyunOSSTestUtils.java | 2 +-
.../fs/s3a/AWSCredentialProviderList.java | 2 +-
.../fs/s3a/BasicAWSCredentialsProvider.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +-
.../apache/hadoop/fs/s3a/S3AInputStream.java | 2 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 2 +-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 2 +-
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 2 +-
.../s3a/auth/AssumedRoleCredentialProvider.java | 2 +-
.../s3a/commit/files/SinglePendingCommit.java | 2 +-
.../hadoop/fs/s3a/commit/files/SuccessData.java | 2 +-
.../fs/s3a/commit/magic/MagicCommitTracker.java | 2 +-
.../hadoop/fs/s3a/commit/staging/Paths.java | 2 +-
.../fs/s3a/s3guard/DynamoDBClientFactory.java | 2 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 2 +-
.../fs/s3a/s3guard/LocalMetadataStore.java | 2 +-
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 2 +-
.../hadoop/fs/s3native/S3xLoginHelper.java | 4 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 4 +-
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 2 +-
...onSSEKMSUserDefinedKeyBlockOutputStream.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 2 +-
.../fs/s3a/commit/AbstractCommitITest.java | 2 +-
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 2 +-
.../hadoop/fs/adl/TestAzureADTokenProvider.java | 2 +-
.../fs/azure/AzureNativeFileSystemStore.java | 2 +-
.../hadoop/fs/azure/BlockBlobAppendStream.java | 2 +-
.../fs/azure/ClientThrottlingAnalyzer.java | 2 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 2 +-
.../hadoop/fs/azure/PageBlobOutputStream.java | 2 +-
.../fs/azure/RemoteWasbAuthorizerImpl.java | 2 +-
.../fs/azure/SecureWasbRemoteCallHelper.java | 2 +-
.../fs/azure/ITestBlobDataValidation.java | 2 +-
.../azure/ITestNativeAzureFileSystemAppend.java | 2 +-
.../hadoop/fs/azure/MockStorageInterface.java | 12 ++--
...estNativeAzureFileSystemBlockCompaction.java | 2 +-
.../fs/azure/integration/AzureTestUtils.java | 2 +-
.../org/apache/hadoop/tools/DistCpOptions.java | 2 +-
.../org/apache/hadoop/tools/OptionsParser.java | 2 +-
.../apache/hadoop/tools/mapred/CopyMapper.java | 2 +-
.../mapred/gridmix/ClusterSummarizer.java | 2 +-
.../mapred/gridmix/ExecutionSummarizer.java | 4 +-
.../hadoop/mapred/gridmix/JobFactory.java | 2 +-
.../mapred/gridmix/RandomTextDataGenerator.java | 2 +-
.../translator/impl/BaseLogParser.java | 1 -
.../WordListAnonymizerUtility.java | 2 +-
.../util/MapReduceJobPropertiesParser.java | 2 +-
.../hadoop/streaming/TestUnconsumedInput.java | 2 +-
102 files changed, 207 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-client-modules/hadoop-client-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
index b9363de..6fa24b4 100644
--- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml
+++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml
@@ -167,10 +167,6 @@
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
@@ -496,10 +492,6 @@
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 1a16dc4..67a5a54 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -157,11 +157,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
index 5a616f7..c5bdf4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationServlet.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.conf;
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
import java.util.Collection;
import java.util.Enumeration;
@@ -72,10 +72,10 @@ public class ReconfigurationServlet extends HttpServlet {
private void printHeader(PrintWriter out, String nodeName) {
out.print("<html><head>");
out.printf("<title>%s Reconfiguration Utility</title>%n",
- StringEscapeUtils.escapeHtml(nodeName));
+ StringEscapeUtils.escapeHtml4(nodeName));
out.print("</head><body>\n");
out.printf("<h1>%s Reconfiguration Utility</h1>%n",
- StringEscapeUtils.escapeHtml(nodeName));
+ StringEscapeUtils.escapeHtml4(nodeName));
}
private void printFooter(PrintWriter out) {
@@ -103,20 +103,20 @@ public class ReconfigurationServlet extends HttpServlet {
out.print("<tr><td>");
if (!reconf.isPropertyReconfigurable(c.prop)) {
out.print("<font color=\"red\">" +
- StringEscapeUtils.escapeHtml(c.prop) + "</font>");
+ StringEscapeUtils.escapeHtml4(c.prop) + "</font>");
changeOK = false;
} else {
- out.print(StringEscapeUtils.escapeHtml(c.prop));
+ out.print(StringEscapeUtils.escapeHtml4(c.prop));
out.print("<input type=\"hidden\" name=\"" +
- StringEscapeUtils.escapeHtml(c.prop) + "\" value=\"" +
- StringEscapeUtils.escapeHtml(c.newVal) + "\"/>");
+ StringEscapeUtils.escapeHtml4(c.prop) + "\" value=\"" +
+ StringEscapeUtils.escapeHtml4(c.newVal) + "\"/>");
}
out.print("</td><td>" +
(c.oldVal == null ? "<it>default</it>" :
- StringEscapeUtils.escapeHtml(c.oldVal)) +
+ StringEscapeUtils.escapeHtml4(c.oldVal)) +
"</td><td>" +
(c.newVal == null ? "<it>default</it>" :
- StringEscapeUtils.escapeHtml(c.newVal)) +
+ StringEscapeUtils.escapeHtml4(c.newVal)) +
"</td>");
out.print("</tr>\n");
}
@@ -147,9 +147,9 @@ public class ReconfigurationServlet extends HttpServlet {
synchronized(oldConf) {
while (params.hasMoreElements()) {
String rawParam = params.nextElement();
- String param = StringEscapeUtils.unescapeHtml(rawParam);
+ String param = StringEscapeUtils.unescapeHtml4(rawParam);
String value =
- StringEscapeUtils.unescapeHtml(req.getParameter(rawParam));
+ StringEscapeUtils.unescapeHtml4(req.getParameter(rawParam));
if (value != null) {
if (value.equals(newConf.getRaw(param)) || value.equals("default") ||
value.equals("null") || value.isEmpty()) {
@@ -157,8 +157,8 @@ public class ReconfigurationServlet extends HttpServlet {
value.isEmpty()) &&
oldConf.getRaw(param) != null) {
out.println("<p>Changed \"" +
- StringEscapeUtils.escapeHtml(param) + "\" from \"" +
- StringEscapeUtils.escapeHtml(oldConf.getRaw(param)) +
+ StringEscapeUtils.escapeHtml4(param) + "\" from \"" +
+ StringEscapeUtils.escapeHtml4(oldConf.getRaw(param)) +
"\" to default</p>");
reconf.reconfigureProperty(param, null);
} else if (!value.equals("default") && !value.equals("null") &&
@@ -168,16 +168,16 @@ public class ReconfigurationServlet extends HttpServlet {
// change from default or value to different value
if (oldConf.getRaw(param) == null) {
out.println("<p>Changed \"" +
- StringEscapeUtils.escapeHtml(param) +
+ StringEscapeUtils.escapeHtml4(param) +
"\" from default to \"" +
- StringEscapeUtils.escapeHtml(value) + "\"</p>");
+ StringEscapeUtils.escapeHtml4(value) + "\"</p>");
} else {
out.println("<p>Changed \"" +
- StringEscapeUtils.escapeHtml(param) + "\" from \"" +
- StringEscapeUtils.escapeHtml(oldConf.
+ StringEscapeUtils.escapeHtml4(param) + "\" from \"" +
+ StringEscapeUtils.escapeHtml4(oldConf.
getRaw(param)) +
"\" to \"" +
- StringEscapeUtils.escapeHtml(value) + "\"</p>");
+ StringEscapeUtils.escapeHtml4(value) + "\"</p>");
}
reconf.reconfigureProperty(param, value);
} else {
@@ -185,10 +185,10 @@ public class ReconfigurationServlet extends HttpServlet {
}
} else {
// parameter value != newConf value
- out.println("<p>\"" + StringEscapeUtils.escapeHtml(param) +
+ out.println("<p>\"" + StringEscapeUtils.escapeHtml4(param) +
"\" not changed because value has changed from \"" +
- StringEscapeUtils.escapeHtml(value) + "\" to \"" +
- StringEscapeUtils.escapeHtml(newConf.getRaw(param)) +
+ StringEscapeUtils.escapeHtml4(value) + "\" to \"" +
+ StringEscapeUtils.escapeHtml4(newConf.getRaw(param)) +
"\" since approval</p>");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 050540b..286312c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -33,8 +33,8 @@ import java.util.Map;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index 9fdf242..fa84c47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -27,7 +27,7 @@ import java.util.Map;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider.Metadata;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 94d3389..5be6e5f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
-import org.apache.commons.lang.WordUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -275,7 +274,7 @@ public class FsShell extends Configured implements Tool {
listing = null;
}
- for (String descLine : WordUtils.wrap(
+ for (String descLine : StringUtils.wrap(
line, MAX_LINE_WIDTH, "\n", true).split("\n")) {
out.println(prefix + descLine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
index 252b3cc..b6244d6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java
@@ -27,7 +27,7 @@ import java.net.URISyntaxException;
import java.util.regex.Pattern;
import org.apache.avro.reflect.Stringable;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 8f6fc4d..011e489 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -23,7 +23,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
index 9dd7771..bbedf2a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.io;
import com.google.common.collect.ComparisonChain;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import java.nio.ByteBuffer;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
index f008e24..0f95058 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
@@ -22,8 +22,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
index 4126344..4d820c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
@@ -21,7 +21,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.FileDescriptor;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index bdfa471..b156d1f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.ipc;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index 0a00ca7..f12ecb6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -39,7 +39,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.AtomicDoubleArray;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -429,7 +429,7 @@ public class DecayRpcScheduler implements RpcScheduler,
updateAverageResponseTime(true);
} catch (Exception ex) {
LOG.error("decayCurrentCounts exception: " +
- ExceptionUtils.getFullStackTrace(ex));
+ ExceptionUtils.getStackTrace(ex));
throw ex;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 6d9ea3e..3a8c83d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.CallQueueManager.CallQueueOverflowException;
import org.apache.hadoop.metrics2.util.MBeans;
@@ -286,7 +286,7 @@ public class FairCallQueue<E extends Schedulable> extends AbstractQueue<E>
*/
@Override
public Iterator<E> iterator() {
- throw new NotImplementedException();
+ throw new NotImplementedException("Code is not implemented");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
index ce6fbe1..1d62c0a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsJsonBuilder.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.metrics2;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.codehaus.jackson.map.ObjectMapper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
index 3d7a90e..9b54adc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MethodMetric.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.metrics2.lib;
import java.lang.reflect.Method;
import static com.google.common.base.Preconditions.*;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
index b2042e7..a3ca98d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableMetricsFactory.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.metrics2.lib;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
index cc32975..6b30618 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
index 053cb55..22c288a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRollingAverages.java
@@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
index 92fe3d1..5ef3178 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.metrics2.lib;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.metrics2.MetricsInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
index 0f6e9a9..92ac952 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/RollingFileSystemSink.java
@@ -37,7 +37,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.configuration2.SubsetConfiguration;
-import org.apache.commons.lang.time.FastDateFormat;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
index ead9a74..45759df 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/TableMapping.java
@@ -28,7 +28,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
index ac118c0..9693220 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocket.java
@@ -28,7 +28,7 @@ import java.nio.channels.ClosedChannelException;
import java.nio.channels.ReadableByteChannel;
import java.nio.ByteBuffer;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.CloseableReferenceCount;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
index c7af97f..e36399f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java
@@ -32,7 +32,7 @@ import java.util.Map;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.util.NativeCodeLoader;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
index 976ddba..31f4398 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index 6085121..0a00d79 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -27,7 +27,7 @@ import java.util.List;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.tools.CommandShell;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
index 34d9fe2..02c168f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
@@ -34,7 +34,7 @@ import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.google.common.annotations.VisibleForTesting;
import java.util.stream.Collectors;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index d36ad9b..f154f2d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -26,7 +26,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.ServiceLoader;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
index bc2e2d4..85015fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/TableListing.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.tools;
import java.util.ArrayList;
import java.util.LinkedList;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.WordUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
/**
@@ -103,7 +102,8 @@ public class TableListing {
// Line-wrap if it's too long
String[] lines = new String[] {raw};
if (wrap) {
- lines = WordUtils.wrap(lines[0], wrapWidth, "\n", true).split("\n");
+ lines = org.apache.hadoop.util.StringUtils.wrap(lines[0], wrapWidth,
+ "\n", true).split("\n");
}
for (int i=0; i<lines.length; i++) {
if (justification == Justification.LEFT) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 33a2010..3db805f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -35,7 +35,7 @@ import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -987,7 +987,7 @@ public class StringUtils {
String[] words = split(StringUtils.toLowerCase(s), ESCAPE_CHAR, '_');
for (String word : words)
- sb.append(org.apache.commons.lang.StringUtils.capitalize(word));
+ sb.append(org.apache.commons.lang3.StringUtils.capitalize(word));
return sb.toString();
}
@@ -1183,4 +1183,64 @@ public class StringUtils {
return true;
}
+ /**
+ * Same as WordUtils#wrap in commons-lang 2.6. Unlike commons-lang3, leading
+ * spaces on the first line are NOT stripped.
+ *
+ * @param str the String to be word wrapped, may be null
+ * @param wrapLength the column to wrap the words at, less than 1 is treated
+ * as 1
+ * @param newLineStr the string to insert for a new line,
+ * <code>null</code> uses the system property line separator
+ * @param wrapLongWords true if long words (such as URLs) should be wrapped
+ * @return a line with newlines inserted, <code>null</code> if null input
+ */
+ public static String wrap(String str, int wrapLength, String newLineStr,
+ boolean wrapLongWords) {
+ if(str == null) {
+ return null;
+ } else {
+ if(newLineStr == null) {
+ newLineStr = System.lineSeparator();
+ }
+
+ if(wrapLength < 1) {
+ wrapLength = 1;
+ }
+
+ int inputLineLength = str.length();
+ int offset = 0;
+ StringBuffer wrappedLine = new StringBuffer(inputLineLength + 32);
+
+ while(inputLineLength - offset > wrapLength) {
+ if(str.charAt(offset) == 32) {
+ ++offset;
+ } else {
+ int spaceToWrapAt = str.lastIndexOf(32, wrapLength + offset);
+ if(spaceToWrapAt >= offset) {
+ wrappedLine.append(str.substring(offset, spaceToWrapAt));
+ wrappedLine.append(newLineStr);
+ offset = spaceToWrapAt + 1;
+ } else if(wrapLongWords) {
+ wrappedLine.append(str.substring(offset, wrapLength + offset));
+ wrappedLine.append(newLineStr);
+ offset += wrapLength;
+ } else {
+ spaceToWrapAt = str.indexOf(32, wrapLength + offset);
+ if(spaceToWrapAt >= 0) {
+ wrappedLine.append(str.substring(offset, spaceToWrapAt));
+ wrappedLine.append(newLineStr);
+ offset = spaceToWrapAt + 1;
+ } else {
+ wrappedLine.append(str.substring(offset));
+ offset = inputLineLength;
+ }
+ }
+ }
+ }
+
+ wrappedLine.append(str.substring(offset));
+ return wrappedLine.toString();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index e865bf1..2361626 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -62,7 +62,7 @@ import static org.apache.hadoop.conf.StorageUnit.TB;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 7f27d7d..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.conf;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
index 6c2e5b8..2ea4523 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/random/TestOsSecureRandom.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.crypto.random;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.junit.Assume;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
index da07105..8cbca8e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSTestWrapper.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.fs;
import java.io.IOException;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.test.GenericTestUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
index 1f37f74..b5307a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.FileNotFoundException;
import java.util.EnumSet;
-import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.CreateOpts.BlockSize;
import org.apache.hadoop.io.IOUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
index 3def5d5..6b9a34c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFCachingGetSpaceUsed.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.fs;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
index 597eb93..fa68264 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs;
-import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.fs.StorageStatistics.LongStatistic;
import org.junit.Before;
@@ -67,15 +67,15 @@ public class TestFileSystemStorageStatistics {
@Before
public void setup() {
- statistics.incrementBytesRead(RandomUtils.nextInt(100));
- statistics.incrementBytesWritten(RandomUtils.nextInt(100));
- statistics.incrementLargeReadOps(RandomUtils.nextInt(100));
- statistics.incrementWriteOps(RandomUtils.nextInt(100));
+ statistics.incrementBytesRead(RandomUtils.nextInt(0, 100));
+ statistics.incrementBytesWritten(RandomUtils.nextInt(0, 100));
+ statistics.incrementLargeReadOps(RandomUtils.nextInt(0, 100));
+ statistics.incrementWriteOps(RandomUtils.nextInt(0, 100));
- statistics.incrementBytesReadByDistance(0, RandomUtils.nextInt(100));
- statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(100));
- statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(100));
- statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(100));
+ statistics.incrementBytesReadByDistance(0, RandomUtils.nextInt(0, 100));
+ statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(0, 100));
+ statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(0, 100));
+ statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(0, 100));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
index 8e60540..e7f36fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopyFromLocal.java
@@ -17,8 +17,8 @@
*/
package org.apache.hadoop.fs.shell;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.math.RandomUtils;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
@@ -56,11 +56,11 @@ public class TestCopyFromLocal {
fs.mkdirs(toDirPath);
int numTotalFiles = 0;
- int numDirs = RandomUtils.nextInt(5);
+ int numDirs = RandomUtils.nextInt(0, 5);
for (int dirCount = 0; dirCount < numDirs; ++dirCount) {
Path subDirPath = new Path(fromDirPath, "subdir" + dirCount);
fs.mkdirs(subDirPath);
- int numFiles = RandomUtils.nextInt(10);
+ int numFiles = RandomUtils.nextInt(0, 10);
for (int fileCount = 0; fileCount < numFiles; ++fileCount) {
numTotalFiles++;
Path subFile = new Path(subDirPath, "file" + fileCount);
@@ -115,7 +115,7 @@ public class TestCopyFromLocal {
Path dir = new Path("dir" + RandomStringUtils.randomNumeric(4));
int numFiles = TestCopyFromLocal.initialize(dir);
int maxThreads = Runtime.getRuntime().availableProcessors() * 2;
- int randThreads = RandomUtils.nextInt(maxThreads - 1) + 1;
+ int randThreads = RandomUtils.nextInt(0, maxThreads - 1) + 1;
String numThreads = Integer.toString(randThreads);
run(new TestMultiThreadedCopy(randThreads,
randThreads == 1 ? 0 : numFiles), "-t", numThreads,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
index fbe3fb8..17be587 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
@@ -26,7 +26,7 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
index 41ae910..5fbd957 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.ipc;
import com.google.protobuf.BlockingService;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.metrics.RpcMetrics;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 0b463a5..5200420 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.ipc;
import com.google.protobuf.ServiceException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index 30176f2..62bd1b1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -38,7 +38,7 @@ import java.util.Enumeration;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.KerberosAuthException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
index 28b3cbe..c0d204f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/unix/TestDomainSocket.java
@@ -43,7 +43,7 @@ import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.unix.DomainSocket.DomainChannel;
import org.apache.hadoop.test.GenericTestUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
index 9357f48..0f8f1e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
@@ -21,7 +21,7 @@ import static org.apache.hadoop.security.token.delegation.web.DelegationTokenAut
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
index 127b0b3..d7c8631 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/service/launcher/AbstractServiceLauncherTestBase.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.service.launcher;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.service.ServiceOperations;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 53eb2be..3e9da1b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
@@ -344,7 +344,7 @@ public abstract class GenericTestUtils {
throw new AssertionError(E_NULL_THROWABLE_STRING, t);
}
if (expectedText != null && !msg.contains(expectedText)) {
- String prefix = org.apache.commons.lang.StringUtils.isEmpty(message)
+ String prefix = org.apache.commons.lang3.StringUtils.isEmpty(message)
? "" : (message + ": ");
throw new AssertionError(
String.format("%s Expected to find '%s' %s: %s",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
index 2aa5e95..d539823 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShutdownHookManager.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.util;
-import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.LoggerFactory;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
index a9fa4c6..b61cebc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSignalLogger.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.util;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.SystemUtils;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index f8e69e2..5992e54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -332,7 +332,7 @@ class FSNamesystemLock {
private static String getMetricName(String operationName, boolean isWrite) {
return (isWrite ? WRITE_LOCK_METRIC_PREFIX : READ_LOCK_METRIC_PREFIX) +
- org.apache.commons.lang.StringUtils.capitalize(operationName) +
+ org.apache.commons.lang3.StringUtils.capitalize(operationName) +
LOCK_METRIC_SUFFIX;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dfd1eac..8e28afe 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1044,11 +1044,6 @@
<version>4.11</version>
</dependency>
<dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>2.6</version>
- </dependency>
- <dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.2</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
index 58c14a9..32367af 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
@@ -22,7 +22,7 @@ import com.aliyun.oss.common.auth.Credentials;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.DefaultCredentials;
import com.aliyun.oss.common.auth.InvalidCredentialsException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 93e31d5..4fbb6fb 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CreateFlag;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index cc050c8..5e21759 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -45,7 +45,7 @@ import com.aliyun.oss.model.UploadPartCopyResult;
import com.aliyun.oss.model.UploadPartRequest;
import com.aliyun.oss.model.UploadPartResult;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index 2fe06c1..a7536d6 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.security.ProviderUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
index 901cb2b..79e0de3 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.fs.aliyun.oss;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.junit.internal.AssumptionViolatedException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
index e0bee0f..10201f0 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java
@@ -23,7 +23,7 @@ import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AnonymousAWSCredentials;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.IOUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
index b1899e2..01bcc6a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.AWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 4b0c208..737d7da 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -121,8 +121,8 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Invoker.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index c54d3e26..440739d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -24,7 +24,7 @@ import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.SSECustomerKey;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.CanSetReadahead;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a01a80..a5f7d75 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -35,7 +35,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
index 9939bb2..7f9e57e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
index e959908..3b89bde 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicSessionCredentials;
import com.amazonaws.auth.AWSCredentials;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.net.URI;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
index 4b6a77e..fdaf9bd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java
@@ -32,7 +32,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
index 85cc38a..596dd95 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
@@ -33,7 +33,7 @@ import java.util.Map;
import com.amazonaws.services.s3.model.PartETag;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
index 6cf1f1e..cf84cb3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
@@ -28,7 +28,7 @@ import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
index cf365c2..a619fc7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
@@ -28,7 +28,7 @@ import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
index a4d39d7..d5d256a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
@@ -30,7 +30,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.UncheckedExecutionException;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2399d6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
index 66ada49..91e64cd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
@@ -26,7 +26,7 @@ import com.amazonaws.regions.Regions;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-8326. Removed exit code file
check for launched container. Contributed by Shane Kumpf
Posted by gi...@apache.org.
YARN-8326. Removed exit code file check for launched container.
Contributed by Shane Kumpf
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a32bc39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a32bc39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a32bc39
Branch: refs/heads/HADOOP-15461
Commit: 8a32bc39eb210fca8052c472601e24c2446b4cc2
Parents: 1cdce86
Author: Eric Yang <ey...@apache.org>
Authored: Fri Jun 22 19:12:48 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Jun 22 19:12:48 2018 -0400
----------------------------------------------------------------------
.../launcher/ContainerLaunch.java | 19 -------------------
1 file changed, 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a32bc39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index bb842af..04295e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -808,25 +808,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
}
- final int sleepMsec = 100;
- int msecLeft = 2000;
- if (pidFilePath != null) {
- File file = new File(getExitCodeFile(pidFilePath.toString()));
- while (!file.exists() && msecLeft >= 0) {
- try {
- Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
- }
- msecLeft -= sleepMsec;
- }
- if (msecLeft < 0) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Timeout while waiting for the exit code file: "
- + file.getAbsolutePath());
- }
- }
- }
-
// Reap the container
boolean result = exec.reapContainer(
new ContainerReapContext.Builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-185:
TestCloseContainerByPipeline#testCloseContainerViaRatis fail intermittently.
Contributed by Shashikant Banerjee.
Posted by gi...@apache.org.
HDDS-185: TestCloseContainerByPipeline#testCloseContainerViaRatis fail intermittently. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/852ca6b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/852ca6b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/852ca6b4
Branch: refs/heads/HADOOP-15461
Commit: 852ca6b432a801df51f957a9b169483008e3c744
Parents: 2911943
Author: Nanda kumar <na...@apache.org>
Authored: Fri Jun 29 04:11:39 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Fri Jun 29 04:11:39 2018 +0530
----------------------------------------------------------------------
.../TestCloseContainerByPipeline.java | 65 +++++++++++++++++---
1 file changed, 58 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/852ca6b4/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index 9e8cb46..265c82b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -33,6 +35,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
@@ -45,6 +48,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeoutException;
public class TestCloseContainerByPipeline {
@@ -89,6 +93,59 @@ public class TestCloseContainerByPipeline {
}
@Test
+ public void testIfCloseContainerCommandHandlerIsInvoked() throws Exception {
+ OzoneOutputStream key = objectStore.getVolume("test").getBucket("test")
+ .createKey("testCloseContainer", 1024, ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE);
+ key.write("standalone".getBytes());
+ key.close();
+
+ //get the name of a valid container
+ KsmKeyArgs keyArgs =
+ new KsmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
+ .setType(HddsProtos.ReplicationType.STAND_ALONE)
+ .setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
+ .setKeyName("testCloseContainer").build();
+
+ KsmKeyLocationInfo ksmKeyLocationInfo =
+ cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
+ .get(0).getBlocksLatestVersionOnly().get(0);
+
+ long containerID = ksmKeyLocationInfo.getContainerID();
+ List<DatanodeDetails> datanodes =
+ cluster.getStorageContainerManager().getContainerInfo(containerID)
+ .getPipeline().getMachines();
+ Assert.assertTrue(datanodes.size() == 1);
+
+ DatanodeDetails datanodeDetails = datanodes.get(0);
+ HddsDatanodeService datanodeService = null;
+ Assert
+ .assertFalse(isContainerClosed(cluster, containerID, datanodeDetails));
+ for (HddsDatanodeService datanodeServiceItr : cluster.getHddsDatanodes()) {
+ if (datanodeDetails.equals(datanodeServiceItr.getDatanodeDetails())) {
+ datanodeService = datanodeServiceItr;
+ break;
+ }
+ }
+ CommandHandler closeContainerHandler =
+ datanodeService.getDatanodeStateMachine().getCommandDispatcher()
+ .getCloseContainerHandler();
+ int lastInvocationCount = closeContainerHandler.getInvocationCount();
+ //send the order to close the container
+ cluster.getStorageContainerManager().getScmNodeManager()
+ .addDatanodeCommand(datanodeDetails.getUuid(),
+ new CloseContainerCommand(containerID,
+ HddsProtos.ReplicationType.STAND_ALONE));
+ GenericTestUtils
+ .waitFor(() -> isContainerClosed(cluster, containerID, datanodeDetails),
+ 500, 5 * 1000);
+ // Make sure the closeContainerCommandHandler is Invoked
+ Assert.assertTrue(
+ closeContainerHandler.getInvocationCount() > lastInvocationCount);
+
+ }
+
+ @Test
public void testCloseContainerViaStandaAlone()
throws IOException, TimeoutException, InterruptedException {
@@ -205,13 +262,7 @@ public class TestCloseContainerByPipeline {
containerData =
datanodeService.getDatanodeStateMachine().getContainer()
.getContainerManager().readContainer(containerID);
- if (!containerData.isOpen()) {
- // make sure the closeContainerHandler on the Datanode is invoked
- Assert.assertTrue(
- datanodeService.getDatanodeStateMachine().getCommandDispatcher()
- .getCloseContainerHandler().getInvocationCount() > 0);
- return true;
- }
+ return !containerData.isOpen();
}
} catch (StorageContainerException e) {
throw new AssertionError(e);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-191. Queue SCMCommands via
EventQueue in SCM. Contributed by Elek, Marton.
Posted by gi...@apache.org.
HDDS-191. Queue SCMCommands via EventQueue in SCM.
Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d6bba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d6bba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d6bba
Branch: refs/heads/HADOOP-15461
Commit: a55d6bba71c81c1c4e9d8cd11f55c78f10a548b0
Parents: 4ffe68a
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Jun 25 13:05:22 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Jun 25 13:05:22 2018 -0700
----------------------------------------------------------------------
.../protocol/commands/CommandForDatanode.java | 45 ++++++++++++++++++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 20 ++++++++-
.../scm/server/StorageContainerManager.java | 8 +++-
.../hadoop/hdds/scm/node/TestNodeManager.java | 39 +++++++++++++++++
4 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
new file mode 100644
index 0000000..0c4964a
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CommandForDatanode.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.protocol.commands;
+
+import java.util.UUID;
+
+import com.google.protobuf.GeneratedMessage;
+
+/**
+ * Command for the datanode with the destination address.
+ */
+public class CommandForDatanode<T extends GeneratedMessage> {
+
+ private final UUID datanodeId;
+
+ private final SCMCommand<T> command;
+
+ public CommandForDatanode(UUID datanodeId, SCMCommand<T> command) {
+ this.datanodeId = datanodeId;
+ this.command = command;
+ }
+
+ public UUID getDatanodeId() {
+ return datanodeId;
+ }
+
+ public SCMCommand<T> getCommand() {
+ return command;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index b339fb7..fc8b013 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.VersionInfo;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.server.events.Event;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.apache.hadoop.hdds.server.events.TypedEvent;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -42,11 +46,14 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
import org.apache.hadoop.ozone.protocol.VersionResponse;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import com.google.protobuf.GeneratedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -100,7 +107,8 @@ import static org.apache.hadoop.util.Time.monotonicNow;
* as soon as you read it.
*/
public class SCMNodeManager
- implements NodeManager, StorageContainerNodeProtocol {
+ implements NodeManager, StorageContainerNodeProtocol,
+ EventHandler<CommandForDatanode> {
@VisibleForTesting
static final Logger LOG =
@@ -154,6 +162,9 @@ public class SCMNodeManager
private final SCMNodePoolManager nodePoolManager;
private final StorageContainerManager scmManager;
+ public static final Event<CommandForDatanode> DATANODE_COMMAND =
+ new TypedEvent<>(CommandForDatanode.class, "DATANODE_COMMAND");
+
/**
* Constructs SCM machine Manager.
*/
@@ -871,4 +882,11 @@ public class SCMNodeManager
public void setStaleNodeIntervalMs(long interval) {
this.staleNodeIntervalMs = interval;
}
+
+ @Override
+ public void onMessage(CommandForDatanode commandForDatanode,
+ EventPublisher publisher) {
+ addDatanodeCommand(commandForDatanode.getDatanodeId(),
+ commandForDatanode.getCommand());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 78f13cb..5725d23 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
+import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RPC;
@@ -51,6 +52,7 @@ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.common.StorageInfo;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
@@ -161,8 +163,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
throw new SCMException("SCM not initialized.", ResultCodes
.SCM_NOT_INITIALIZED);
}
+ EventQueue eventQueue = new EventQueue();
+
+ SCMNodeManager nm = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
+ scmNodeManager = nm;
+ eventQueue.addHandler(SCMNodeManager.DATANODE_COMMAND, nm);
- scmNodeManager = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
scmContainerManager = new ContainerMapping(conf, getScmNodeManager(),
cacheSize);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d6bba/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index 2b04d6b..824a135 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.base.Supplier;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -29,7 +30,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
@@ -1165,4 +1169,39 @@ public class TestNodeManager {
assertEquals(expectedRemaining, foundRemaining);
}
}
+
+ @Test
+ public void testHandlingSCMCommandEvent() {
+ OzoneConfiguration conf = getConf();
+ conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ 100, TimeUnit.MILLISECONDS);
+
+ DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
+ String dnId = datanodeDetails.getUuidString();
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ List<StorageReportProto> reports =
+ TestUtils.createStorageReport(100, 10, 90,
+ storagePath, null, dnId, 1);
+
+ EventQueue eq = new EventQueue();
+ try (SCMNodeManager nodemanager = createNodeManager(conf)) {
+ eq.addHandler(SCMNodeManager.DATANODE_COMMAND, nodemanager);
+
+ nodemanager
+ .register(datanodeDetails, TestUtils.createNodeReport(reports));
+ eq.fireEvent(SCMNodeManager.DATANODE_COMMAND,
+ new CommandForDatanode(datanodeDetails.getUuid(),
+ new CloseContainerCommand(1L, ReplicationType.STAND_ALONE)));
+
+ eq.processAll(1000L);
+ List<SCMCommand> command =
+ nodemanager.sendHeartbeat(datanodeDetails, null);
+ Assert.assertEquals(1, command.size());
+ Assert
+ .assertEquals(command.get(0).getClass(), CloseContainerCommand.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: Revert "HDDS-194. Remove
NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"
Posted by gi...@apache.org.
Revert "HDDS-194. Remove NodePoolManager and node pool handling from SCM. Contributed by Elek Marton"
This reverts commit aaf03cc459a34af284f9735453aefd4ddb430d67.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d6fe5f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d6fe5f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d6fe5f3
Branch: refs/heads/HADOOP-15461
Commit: 0d6fe5f36be5b19aab89d995866e526c5feec758
Parents: aaf03cc
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Jun 27 13:25:45 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Jun 27 13:25:45 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 26 ++
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 47 +++
.../container/replication/ReplicationQueue.java | 78 -----
.../replication/ReplicationReqMsg.java | 107 ------
.../container/replication/package-info.java | 23 --
.../replication/TestReplicationQueue.java | 134 --------
.../container/replication/package-info.java | 23 --
.../hdds/scm/container/ContainerMapping.java | 10 +-
.../replication/ContainerSupervisor.java | 340 +++++++++++++++++++
.../container/replication/InProgressPool.java | 255 ++++++++++++++
.../scm/container/replication/PeriodicPool.java | 119 +++++++
.../scm/container/replication/package-info.java | 23 ++
.../hadoop/hdds/scm/node/NodeManager.java | 6 +
.../hadoop/hdds/scm/node/NodePoolManager.java | 71 ++++
.../hadoop/hdds/scm/node/SCMNodeManager.java | 23 ++
.../hdds/scm/node/SCMNodePoolManager.java | 269 +++++++++++++++
.../hdds/scm/container/MockNodeManager.java | 6 +
.../hdds/scm/node/TestSCMNodePoolManager.java | 160 +++++++++
.../testutils/ReplicationNodeManagerMock.java | 5 +
.../ReplicationNodePoolManagerMock.java | 133 ++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 31 ++
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 74 ++++
23 files changed, 1596 insertions(+), 368 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index df6fbf0..85407e6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -243,6 +243,32 @@ public final class ScmConfigKeys {
public static final String
OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
+ /**
+ * Don't start processing a pool if we have not had a minimum number of
+ * seconds from the last processing.
+ */
+ public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
+ "ozone.scm.container.report.processing.interval";
+ public static final String
+ OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
+
+ /**
+ * This determines the total number of pools to be processed in parallel.
+ */
+ public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
+ "ozone.scm.max.nodepool.processing.threads";
+ public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
+ /**
+ * These 2 settings control the number of threads in executor pool and time
+ * outs for thw container reports from all nodes.
+ */
+ public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
+ "ozone.scm.max.container.report.threads";
+ public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
+ public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
+ "ozone.scm.container.reports.wait.timeout";
+ public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
+ "5m";
public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
"ozone.scm.block.deletion.max.retry";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 08a5ffd..c40dc8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -91,6 +91,7 @@ public final class OzoneConsts {
public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String BLOCK_DB = "block.db";
+ public static final String NODEPOOL_DB = "nodepool.db";
public static final String OPEN_CONTAINERS_DB = "openContainers.db";
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String KSM_DB_NAME = "ksm.db";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 25365c8..7a91610 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -572,6 +572,25 @@
</description>
</property>
<property>
+ <name>ozone.scm.container.report.processing.interval</name>
+ <value>60s</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>Time interval for scm to process container reports
+ for a node pool. Scm handles node pool reports in a cyclic clock
+ manner, it fetches pools periodically with this time interval.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.container.reports.wait.timeout</name>
+ <value>300s</value>
+ <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
+ <description>Maximum time to wait in seconds for processing all container
+ reports from
+ a node pool. It determines the timeout for a
+ node pool report.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.container.size.gb</name>
<value>5</value>
<tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
@@ -774,6 +793,17 @@
</description>
</property>
<property>
+ <name>ozone.scm.max.container.report.threads</name>
+ <value>100</value>
+ <tag>OZONE, PERFORMANCE</tag>
+ <description>
+ Maximum number of threads to process container reports in scm.
+ Each container report from a data node is processed by scm in a worker
+ thread, fetched from a thread pool. This property is used to control the
+ maximum size of the thread pool.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.max.hb.count.to.process</name>
<value>5000</value>
<tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
@@ -785,6 +815,14 @@
</description>
</property>
<property>
+ <name>ozone.scm.max.nodepool.processing.threads</name>
+ <value>1</value>
+ <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
+ <description>
+ Number of node pools to process in parallel.
+ </description>
+ </property>
+ <property>
<name>ozone.scm.names</name>
<value/>
<tag>OZONE</tag>
@@ -806,6 +844,15 @@
</description>
</property>
<property>
+ <name>ozone.scm.max.nodepool.processing.threads</name>
+ <value>1</value>
+ <tag>OZONE, SCM</tag>
+ <description>
+ Controls the number of node pools that can be processed in parallel by
+ Container Supervisor.
+ </description>
+ </property>
+ <property>
<name>ozone.trace.enabled</name>
<value>false</value>
<tag>OZONE, DEBUG</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
deleted file mode 100644
index b83ecf1..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationQueue.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Queue;
-
-/**
- * Priority queue to handle under-replicated and over replicated containers
- * in ozone. ReplicationManager will consume these messages and decide
- * accordingly.
- */
-public class ReplicationQueue {
-
- private final Queue<ReplicationReqMsg> queue;
-
- ReplicationQueue() {
- queue = new PriorityQueue<>();
- }
-
- public synchronized boolean add(ReplicationReqMsg repObj) {
- if (this.queue.contains(repObj)) {
- // Remove the earlier message and insert this one
- this.queue.remove(repObj);
- return this.queue.add(repObj);
- } else {
- return this.queue.add(repObj);
- }
- }
-
- public synchronized boolean remove(ReplicationReqMsg repObj) {
- return queue.remove(repObj);
- }
-
- /**
- * Retrieves, but does not remove, the head of this queue,
- * or returns {@code null} if this queue is empty.
- *
- * @return the head of this queue, or {@code null} if this queue is empty
- */
- public synchronized ReplicationReqMsg peek() {
- return queue.peek();
- }
-
- /**
- * Retrieves and removes the head of this queue,
- * or returns {@code null} if this queue is empty.
- *
- * @return the head of this queue, or {@code null} if this queue is empty
- */
- public synchronized ReplicationReqMsg poll() {
- return queue.poll();
- }
-
- public synchronized boolean removeAll(List<ReplicationReqMsg> repObjs) {
- return queue.removeAll(repObjs);
- }
-
- public int size() {
- return queue.size();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
deleted file mode 100644
index 8d26fc3..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationReqMsg.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.io.Serializable;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.math.NumberUtils;
-
-/**
- * Wrapper class for hdds replication queue. Implements its natural
- * ordering for priority queue.
- */
-public class ReplicationReqMsg implements Comparable<ReplicationReqMsg>,
- Serializable {
- private final long containerId;
- private final short replicationCount;
- private final short expecReplicationCount;
- private final long timestamp;
-
- public ReplicationReqMsg(long containerId, short replicationCount,
- long timestamp, short expecReplicationCount) {
- this.containerId = containerId;
- this.replicationCount = replicationCount;
- this.timestamp = timestamp;
- this.expecReplicationCount = expecReplicationCount;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object
- * is less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(ReplicationReqMsg o) {
- if (this == o) {
- return 0;
- }
- if (o == null) {
- return 1;
- }
- int retVal = NumberUtils
- .compare(getReplicationCount() - getExpecReplicationCount(),
- o.getReplicationCount() - o.getExpecReplicationCount());
- if (retVal != 0) {
- return retVal;
- }
- return NumberUtils.compare(getTimestamp(), o.getTimestamp());
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(91, 1011)
- .append(getContainerId())
- .toHashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ReplicationReqMsg that = (ReplicationReqMsg) o;
- return new EqualsBuilder().append(getContainerId(), that.getContainerId())
- .isEquals();
- }
-
- public long getContainerId() {
- return containerId;
- }
-
- public short getReplicationCount() {
- return replicationCount;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public short getExpecReplicationCount() {
- return expecReplicationCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
deleted file mode 100644
index 7f335e3..0000000
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.replication;
-
-/**
- * Ozone Container replicaton related classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
deleted file mode 100644
index 39c61d3..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationQueue.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.replication;
-
-import java.util.Random;
-import java.util.UUID;
-import org.apache.hadoop.util.Time;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for ReplicationQueue.
- */
-public class TestReplicationQueue {
-
- private ReplicationQueue replicationQueue;
- private Random random;
-
- @Before
- public void setUp() {
- replicationQueue = new ReplicationQueue();
- random = new Random();
- }
-
- @Test
- public void testDuplicateAddOp() {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationReqMsg obj1, obj2, obj3;
- long time = Time.monotonicNow();
- obj1 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
- obj2 = new ReplicationReqMsg(contId, (short) 2, time + 1, (short) 3);
- obj3 = new ReplicationReqMsg(contId, (short) 1, time+2, (short) 3);
-
- replicationQueue.add(obj1);
- replicationQueue.add(obj2);
- replicationQueue.add(obj3);
- Assert.assertEquals("Should add only 1 msg as second one is duplicate",
- 1, replicationQueue.size());
- ReplicationReqMsg temp = replicationQueue.poll();
- Assert.assertEquals(temp, obj3);
- }
-
- @Test
- public void testPollOp() {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationReqMsg msg1, msg2, msg3, msg4, msg5;
- msg1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
- (short) 3);
- long time = Time.monotonicNow();
- msg2 = new ReplicationReqMsg(contId + 1, (short) 4, time, (short) 3);
- msg3 = new ReplicationReqMsg(contId + 2, (short) 0, time, (short) 3);
- msg4 = new ReplicationReqMsg(contId, (short) 2, time, (short) 3);
- // Replication message for same container but different nodeId
- msg5 = new ReplicationReqMsg(contId + 1, (short) 2, time, (short) 3);
-
- replicationQueue.add(msg1);
- replicationQueue.add(msg2);
- replicationQueue.add(msg3);
- replicationQueue.add(msg4);
- replicationQueue.add(msg5);
- Assert.assertEquals("Should have 3 objects",
- 3, replicationQueue.size());
-
- // Since Priority queue orders messages according to replication count,
- // message with lowest replication should be first
- ReplicationReqMsg temp;
- temp = replicationQueue.poll();
- Assert.assertEquals("Should have 2 objects",
- 2, replicationQueue.size());
- Assert.assertEquals(temp, msg3);
-
- temp = replicationQueue.poll();
- Assert.assertEquals("Should have 1 objects",
- 1, replicationQueue.size());
- Assert.assertEquals(temp, msg5);
-
- // Message 2 should be ordered before message 5 as both have same replication
- // number but message 2 has earlier timestamp.
- temp = replicationQueue.poll();
- Assert.assertEquals("Should have 0 objects",
- replicationQueue.size(), 0);
- Assert.assertEquals(temp, msg4);
- }
-
- @Test
- public void testRemoveOp() {
- long contId = random.nextLong();
- String nodeId = UUID.randomUUID().toString();
- ReplicationReqMsg obj1, obj2, obj3;
- obj1 = new ReplicationReqMsg(contId, (short) 1, Time.monotonicNow(),
- (short) 3);
- obj2 = new ReplicationReqMsg(contId + 1, (short) 2, Time.monotonicNow(),
- (short) 3);
- obj3 = new ReplicationReqMsg(contId + 2, (short) 3, Time.monotonicNow(),
- (short) 3);
-
- replicationQueue.add(obj1);
- replicationQueue.add(obj2);
- replicationQueue.add(obj3);
- Assert.assertEquals("Should have 3 objects",
- 3, replicationQueue.size());
-
- replicationQueue.remove(obj3);
- Assert.assertEquals("Should have 2 objects",
- 2, replicationQueue.size());
-
- replicationQueue.remove(obj2);
- Assert.assertEquals("Should have 1 objects",
- 1, replicationQueue.size());
-
- replicationQueue.remove(obj1);
- Assert.assertEquals("Should have 0 objects",
- 0, replicationQueue.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
deleted file mode 100644
index 5b1fd0f..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-/**
- * SCM Testing and Mocking Utils.
- */
-package org.apache.hadoop.ozone.container.replication;
-// Test classes for Replication functionality.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 9fd30f2..b563e90 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.closer.ContainerCloser;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerSupervisor;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
@@ -79,6 +80,7 @@ public class ContainerMapping implements Mapping {
private final PipelineSelector pipelineSelector;
private final ContainerStateManager containerStateManager;
private final LeaseManager<ContainerInfo> containerLeaseManager;
+ private final ContainerSupervisor containerSupervisor;
private final float containerCloseThreshold;
private final ContainerCloser closer;
private final long size;
@@ -125,7 +127,9 @@ public class ContainerMapping implements Mapping {
OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
this.containerStateManager =
new ContainerStateManager(conf, this);
-
+ this.containerSupervisor =
+ new ContainerSupervisor(conf, nodeManager,
+ nodeManager.getNodePoolManager());
this.containerCloseThreshold = conf.getFloat(
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
@@ -403,8 +407,8 @@ public class ContainerMapping implements Mapping {
throws IOException {
List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
containerInfos = reports.getReportsList();
-
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+ containerSupervisor.handleContainerReport(datanodeDetails, reports);
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
containerInfos) {
byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
lock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
new file mode 100644
index 0000000..5bd0574
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ContainerSupervisor.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static com.google.common.util.concurrent.Uninterruptibles
+ .sleepUninterruptibly;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
+
+/**
+ * This class takes a set of container reports that belong to a pool and then
+ * computes the replication levels for each container.
+ */
+public class ContainerSupervisor implements Closeable {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ContainerSupervisor.class);
+
+ private final NodePoolManager poolManager;
+ private final HashSet<String> poolNames;
+ private final PriorityQueue<PeriodicPool> poolQueue;
+ private final NodeManager nodeManager;
+ private final long containerProcessingLag;
+ private final AtomicBoolean runnable;
+ private final ExecutorService executorService;
+ private final long maxPoolWait;
+ private long poolProcessCount;
+ private final List<InProgressPool> inProgressPoolList;
+ private final AtomicInteger threadFaultCount;
+ private final int inProgressPoolMaxCount;
+
+ private final ReadWriteLock inProgressPoolListLock;
+
+ /**
+ * Returns the number of times we have processed pools.
+ * @return long
+ */
+ public long getPoolProcessCount() {
+ return poolProcessCount;
+ }
+
+
+ /**
+ * Constructs a class that computes Replication Levels.
+ *
+ * @param conf - OzoneConfiguration
+ * @param nodeManager - Node Manager
+ * @param poolManager - Pool Manager
+ */
+ public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
+ NodePoolManager poolManager) {
+ Preconditions.checkNotNull(poolManager);
+ Preconditions.checkNotNull(nodeManager);
+ this.containerProcessingLag =
+ conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
+ OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS
+ ) * 1000;
+ int maxContainerReportThreads =
+ conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
+ OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
+ );
+ this.maxPoolWait =
+ conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
+ OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ this.inProgressPoolMaxCount = conf.getInt(
+ OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
+ OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
+ this.poolManager = poolManager;
+ this.nodeManager = nodeManager;
+ this.poolNames = new HashSet<>();
+ this.poolQueue = new PriorityQueue<>();
+ this.runnable = new AtomicBoolean(true);
+ this.threadFaultCount = new AtomicInteger(0);
+ this.executorService = newCachedThreadPool(
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("Container Reports Processing Thread - %d")
+ .build(), maxContainerReportThreads);
+ this.inProgressPoolList = new LinkedList<>();
+ this.inProgressPoolListLock = new ReentrantReadWriteLock();
+
+ initPoolProcessThread();
+ }
+
+ private ExecutorService newCachedThreadPool(ThreadFactory threadFactory,
+ int maxThreads) {
+ return new HadoopThreadPoolExecutor(0, maxThreads, 60L, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(), threadFactory);
+ }
+
+ /**
+ * Returns the number of pools that are under process right now.
+ * @return int - Number of pools that are in process.
+ */
+ public int getInProgressPoolCount() {
+ return inProgressPoolList.size();
+ }
+
+ /**
+ * Exits the background thread.
+ */
+ public void setExit() {
+ this.runnable.set(false);
+ }
+
+ /**
+ * Adds or removes pools from names that we need to process.
+ *
+ * There are two different cases that we need to process.
+ * The case where some pools are being added and some times we have to
+ * handle cases where pools are removed.
+ */
+ private void refreshPools() {
+ List<String> pools = this.poolManager.getNodePools();
+ if (pools != null) {
+
+ HashSet<String> removedPools =
+ computePoolDifference(this.poolNames, new HashSet<>(pools));
+
+ HashSet<String> addedPools =
+ computePoolDifference(new HashSet<>(pools), this.poolNames);
+ // TODO: Support remove pool API in pool manager so that this code
+ // path can be tested. This never happens in the current code base.
+ for (String poolName : removedPools) {
+ for (PeriodicPool periodicPool : poolQueue) {
+ if (periodicPool.getPoolName().compareTo(poolName) == 0) {
+ poolQueue.remove(periodicPool);
+ }
+ }
+ }
+ // Remove the pool names that we have in the list.
+ this.poolNames.removeAll(removedPools);
+
+ for (String poolName : addedPools) {
+ poolQueue.add(new PeriodicPool(poolName));
+ }
+
+ // Add to the pool names we are tracking.
+ poolNames.addAll(addedPools);
+ }
+
+ }
+
+ /**
+ * Handle the case where pools are added.
+ *
+ * @param newPools - New Pools list
+ * @param oldPool - oldPool List.
+ */
+ private HashSet<String> computePoolDifference(HashSet<String> newPools,
+ Set<String> oldPool) {
+ Preconditions.checkNotNull(newPools);
+ Preconditions.checkNotNull(oldPool);
+ HashSet<String> newSet = new HashSet<>(newPools);
+ newSet.removeAll(oldPool);
+ return newSet;
+ }
+
+ private void initPoolProcessThread() {
+
+ /*
+ * Task that runs to check if we need to start a pool processing job.
+ * if so we create a pool reconciliation job and find out of all the
+ * expected containers are on the nodes.
+ */
+ Runnable processPools = () -> {
+ while (runnable.get()) {
+ // Make sure that we don't have any new pools.
+ refreshPools();
+ while (inProgressPoolList.size() < inProgressPoolMaxCount) {
+ PeriodicPool pool = poolQueue.poll();
+ if (pool != null) {
+ if (pool.getLastProcessedTime() + this.containerProcessingLag >
+ Time.monotonicNow()) {
+ LOG.debug("Not within the time window for processing: {}",
+ pool.getPoolName());
+ // we might over sleep here, not a big deal.
+ sleepUninterruptibly(this.containerProcessingLag,
+ TimeUnit.MILLISECONDS);
+ }
+ LOG.debug("Adding pool {} to container processing queue",
+ pool.getPoolName());
+ InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
+ pool, this.nodeManager, this.poolManager, this.executorService);
+ inProgressPool.startReconciliation();
+ inProgressPoolListLock.writeLock().lock();
+ try {
+ inProgressPoolList.add(inProgressPool);
+ } finally {
+ inProgressPoolListLock.writeLock().unlock();
+ }
+ poolProcessCount++;
+ } else {
+ break;
+ }
+ }
+ sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
+ inProgressPoolListLock.readLock().lock();
+ try {
+ for (InProgressPool inProgressPool : inProgressPoolList) {
+ inProgressPool.finalizeReconciliation();
+ poolQueue.add(inProgressPool.getPool());
+ }
+ } finally {
+ inProgressPoolListLock.readLock().unlock();
+ }
+ inProgressPoolListLock.writeLock().lock();
+ try {
+ inProgressPoolList.clear();
+ } finally {
+ inProgressPoolListLock.writeLock().unlock();
+ }
+ }
+ };
+
+ // We will have only one thread for pool processing.
+ Thread poolProcessThread = new Thread(processPools);
+ poolProcessThread.setDaemon(true);
+ poolProcessThread.setName("Pool replica thread");
+ poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
+ // Let us just restart this thread after logging a critical error.
+ // if this thread is not running we cannot handle commands from SCM.
+ LOG.error("Critical Error : Pool replica thread encountered an " +
+ "error. Thread: {} Error Count : {}", t.toString(), e,
+ threadFaultCount.incrementAndGet());
+ poolProcessThread.start();
+ // TODO : Add a config to restrict how many times we will restart this
+ // thread in a single session.
+ });
+ poolProcessThread.start();
+ }
+
+ /**
+ * Adds a container report to appropriate inProgress Pool.
+ * @param containerReport -- Container report for a specific container from
+ * a datanode.
+ */
+ public void handleContainerReport(DatanodeDetails datanodeDetails,
+ ContainerReportsProto containerReport) {
+ inProgressPoolListLock.readLock().lock();
+ try {
+ String poolName = poolManager.getNodePool(datanodeDetails);
+ for (InProgressPool ppool : inProgressPoolList) {
+ if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
+ ppool.handleContainerReport(datanodeDetails, containerReport);
+ return;
+ }
+ }
+ // TODO: Decide if we can do anything else with this report.
+ LOG.debug("Discarding the container report for pool {}. " +
+ "That pool is not currently in the pool reconciliation process." +
+ " Container Name: {}", poolName, datanodeDetails);
+ } catch (SCMException e) {
+ LOG.warn("Skipping processing container report from datanode {}, "
+ + "cause: failed to get the corresponding node pool",
+ datanodeDetails.toString(), e);
+ } finally {
+ inProgressPoolListLock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Get in process pool list, used for testing.
+ * @return List of InProgressPool
+ */
+ @VisibleForTesting
+ public List<InProgressPool> getInProcessPoolList() {
+ return inProgressPoolList;
+ }
+
+ /**
+ * Shutdown the Container Replication Manager.
+ * @throws IOException if an I/O error occurs
+ */
+ @Override
+ public void close() throws IOException {
+ setExit();
+ HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
new file mode 100644
index 0000000..4b54731
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/InProgressPool.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerInfo;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * These are pools that are actively checking for replication status of the
+ * containers.
+ */
+public final class InProgressPool {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(InProgressPool.class);
+
+ private final PeriodicPool pool;
+ private final NodeManager nodeManager;
+ private final NodePoolManager poolManager;
+ private final ExecutorService executorService;
+ private final Map<Long, Integer> containerCountMap;
+ private final Map<UUID, Boolean> processedNodeSet;
+ private final long startTime;
+ private ProgressStatus status;
+ private AtomicInteger nodeCount;
+ private AtomicInteger nodeProcessed;
+ private AtomicInteger containerProcessedCount;
+ private long maxWaitTime;
+ /**
+ * Constructs an pool that is being processed.
+ * @param maxWaitTime - Maximum wait time in milliseconds.
+ * @param pool - Pool that we are working against
+ * @param nodeManager - Nodemanager
+ * @param poolManager - pool manager
+ * @param executorService - Shared Executor service.
+ */
+ InProgressPool(long maxWaitTime, PeriodicPool pool,
+ NodeManager nodeManager, NodePoolManager poolManager,
+ ExecutorService executorService) {
+ Preconditions.checkNotNull(pool);
+ Preconditions.checkNotNull(nodeManager);
+ Preconditions.checkNotNull(poolManager);
+ Preconditions.checkNotNull(executorService);
+ Preconditions.checkArgument(maxWaitTime > 0);
+ this.pool = pool;
+ this.nodeManager = nodeManager;
+ this.poolManager = poolManager;
+ this.executorService = executorService;
+ this.containerCountMap = new ConcurrentHashMap<>();
+ this.processedNodeSet = new ConcurrentHashMap<>();
+ this.maxWaitTime = maxWaitTime;
+ startTime = Time.monotonicNow();
+ }
+
+ /**
+ * Returns periodic pool.
+ *
+ * @return PeriodicPool
+ */
+ public PeriodicPool getPool() {
+ return pool;
+ }
+
+ /**
+ * We are done if we have got reports from all nodes or we have
+ * done waiting for the specified time.
+ *
+ * @return true if we are done, false otherwise.
+ */
+ public boolean isDone() {
+ return (nodeCount.get() == nodeProcessed.get()) ||
+ (this.startTime + this.maxWaitTime) > Time.monotonicNow();
+ }
+
+ /**
+ * Gets the number of containers processed.
+ *
+ * @return int
+ */
+ public int getContainerProcessedCount() {
+ return containerProcessedCount.get();
+ }
+
+ /**
+ * Returns the start time in milliseconds.
+ *
+ * @return - Start Time.
+ */
+ public long getStartTime() {
+ return startTime;
+ }
+
+ /**
+ * Get the number of nodes in this pool.
+ *
+ * @return - node count
+ */
+ public int getNodeCount() {
+ return nodeCount.get();
+ }
+
+ /**
+ * Get the number of nodes that we have already processed container reports
+ * from.
+ *
+ * @return - Processed count.
+ */
+ public int getNodeProcessed() {
+ return nodeProcessed.get();
+ }
+
+ /**
+ * Returns the current status.
+ *
+ * @return Status
+ */
+ public ProgressStatus getStatus() {
+ return status;
+ }
+
+ /**
+ * Starts the reconciliation process for all the nodes in the pool.
+ */
+ public void startReconciliation() {
+ List<DatanodeDetails> datanodeDetailsList =
+ this.poolManager.getNodes(pool.getPoolName());
+ if (datanodeDetailsList.size() == 0) {
+ LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
+ pool.getPoolName());
+ this.status = ProgressStatus.Error;
+ return;
+ }
+
+ nodeProcessed = new AtomicInteger(0);
+ containerProcessedCount = new AtomicInteger(0);
+ nodeCount = new AtomicInteger(0);
+ this.status = ProgressStatus.InProgress;
+ this.getPool().setLastProcessedTime(Time.monotonicNow());
+ }
+
+ /**
+ * Queues a container Report for handling. This is done in a worker thread
+ * since decoding a container report might be compute intensive . We don't
+ * want to block since we have asked for bunch of container reports
+ * from a set of datanodes.
+ *
+ * @param containerReport - ContainerReport
+ */
+ public void handleContainerReport(DatanodeDetails datanodeDetails,
+ ContainerReportsProto containerReport) {
+ if (status == ProgressStatus.InProgress) {
+ executorService.submit(processContainerReport(datanodeDetails,
+ containerReport));
+ } else {
+ LOG.debug("Cannot handle container report when the pool is in {} status.",
+ status);
+ }
+ }
+
+ private Runnable processContainerReport(DatanodeDetails datanodeDetails,
+ ContainerReportsProto reports) {
+ return () -> {
+ if (processedNodeSet.computeIfAbsent(datanodeDetails.getUuid(),
+ (k) -> true)) {
+ nodeProcessed.incrementAndGet();
+ LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
+ datanodeDetails.getUuid());
+ for (ContainerInfo info : reports.getReportsList()) {
+ containerProcessedCount.incrementAndGet();
+ LOG.debug("Total Containers processed: {} Container Name: {}",
+ containerProcessedCount.get(), info.getContainerID());
+
+ // Update the container map with count + 1 if the key exists or
+ // update the map with 1. Since this is a concurrentMap the
+ // computation and update is atomic.
+ containerCountMap.merge(info.getContainerID(), 1, Integer::sum);
+ }
+ }
+ };
+ }
+
+ /**
+ * Filter the containers based on specific rules.
+ *
+ * @param predicate -- Predicate to filter by
+ * @return A list of map entries.
+ */
+ public List<Map.Entry<Long, Integer>> filterContainer(
+ Predicate<Map.Entry<Long, Integer>> predicate) {
+ return containerCountMap.entrySet().stream()
+ .filter(predicate).collect(Collectors.toList());
+ }
+
+ /**
+ * Used only for testing, calling this will abort container report
+ * processing. This is very dangerous call and should not be made by any users
+ */
+ @VisibleForTesting
+ public void setDoneProcessing() {
+ nodeProcessed.set(nodeCount.get());
+ }
+
+ /**
+ * Returns the pool name.
+ *
+ * @return Name of the pool.
+ */
+ String getPoolName() {
+ return pool.getPoolName();
+ }
+
+ public void finalizeReconciliation() {
+ status = ProgressStatus.Done;
+ //TODO: Add finalizing logic. This is where actual reconciliation happens.
+ }
+
+ /**
+ * Current status of the computing replication status.
+ */
+ public enum ProgressStatus {
+ InProgress, Done, Error
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
new file mode 100644
index 0000000..ef28aa7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/PeriodicPool.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Periodic pool is a pool with a time stamp, this allows us to process pools
+ * based on a cyclic clock.
+ */
+public class PeriodicPool implements Comparable<PeriodicPool> {
+ private final String poolName;
+ private long lastProcessedTime;
+ private AtomicLong totalProcessedCount;
+
+ /**
+ * Constructs a periodic pool.
+ *
+ * @param poolName - Name of the pool
+ */
+ public PeriodicPool(String poolName) {
+ this.poolName = poolName;
+ lastProcessedTime = 0;
+ totalProcessedCount = new AtomicLong(0);
+ }
+
+ /**
+ * Get pool Name.
+ * @return PoolName
+ */
+ public String getPoolName() {
+ return poolName;
+ }
+
+ /**
+ * Compares this object with the specified object for order. Returns a
+ * negative integer, zero, or a positive integer as this object is less
+ * than, equal to, or greater than the specified object.
+ *
+ * @param o the object to be compared.
+ * @return a negative integer, zero, or a positive integer as this object is
+ * less than, equal to, or greater than the specified object.
+ * @throws NullPointerException if the specified object is null
+ * @throws ClassCastException if the specified object's type prevents it
+ * from being compared to this object.
+ */
+ @Override
+ public int compareTo(PeriodicPool o) {
+ return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ PeriodicPool that = (PeriodicPool) o;
+
+ return poolName.equals(that.poolName);
+ }
+
+ @Override
+ public int hashCode() {
+ return poolName.hashCode();
+ }
+
+ /**
+ * Returns the Total Times we have processed this pool.
+ *
+ * @return processed count.
+ */
+ public long getTotalProcessedCount() {
+ return totalProcessedCount.get();
+ }
+
+ /**
+ * Gets the last time we processed this pool.
+ * @return time in milliseconds
+ */
+ public long getLastProcessedTime() {
+ return this.lastProcessedTime;
+ }
+
+
+ /**
+ * Sets the last processed time.
+ *
+ * @param lastProcessedTime - Long in milliseconds.
+ */
+
+ public void setLastProcessedTime(long lastProcessedTime) {
+ this.lastProcessedTime = lastProcessedTime;
+ }
+
+ /*
+ * Increments the total processed count.
+ */
+ public void incTotalProcessedCount() {
+ this.totalProcessedCount.incrementAndGet();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
new file mode 100644
index 0000000..7bbe2ef
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+/*
+ This package contains routines that manage replication of a container. This
+ relies on container reports to understand the replication level of a
+ container - UnderReplicated, Replicated, OverReplicated -- and manages the
+ replication level based on that.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
index 72d7e94..4392633 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java
@@ -124,6 +124,12 @@ public interface NodeManager extends StorageContainerNodeProtocol,
SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails);
/**
+ * Returns the NodePoolManager associated with the NodeManager.
+ * @return NodePoolManager
+ */
+ NodePoolManager getNodePoolManager();
+
+ /**
* Wait for the heartbeat is processed by NodeManager.
* @return true if heartbeat has been processed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
new file mode 100644
index 0000000..46faf9ca
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodePoolManager.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Interface that defines SCM NodePoolManager.
+ */
+public interface NodePoolManager extends Closeable {
+
+ /**
+ * Add a node to a node pool.
+ * @param pool - name of the node pool.
+ * @param node - data node.
+ */
+ void addNode(String pool, DatanodeDetails node) throws IOException;
+
+ /**
+ * Remove a node from a node pool.
+ * @param pool - name of the node pool.
+ * @param node - data node.
+ * @throws SCMException
+ */
+ void removeNode(String pool, DatanodeDetails node)
+ throws SCMException;
+
+ /**
+ * Get a list of known node pools.
+ * @return a list of known node pool names or an empty list if not node pool
+ * is defined.
+ */
+ List<String> getNodePools();
+
+ /**
+ * Get all nodes of a node pool given the name of the node pool.
+ * @param pool - name of the node pool.
+ * @return a list of datanode ids or an empty list if the node pool was not
+ * found.
+ */
+ List<DatanodeDetails> getNodes(String pool);
+
+ /**
+ * Get the node pool name if the node has been added to a node pool.
+ * @param datanodeDetails - datanode ID.
+ * @return node pool name if it has been assigned.
+ * null if the node has not been assigned to any node pool yet.
+ */
+ String getNodePool(DatanodeDetails datanodeDetails) throws SCMException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index adca8ea..fc8b013 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import com.google.protobuf.GeneratedMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -158,6 +159,7 @@ public class SCMNodeManager
private ObjectName nmInfoBean;
// Node pool manager.
+ private final SCMNodePoolManager nodePoolManager;
private final StorageContainerManager scmManager;
public static final Event<CommandForDatanode> DATANODE_COMMAND =
@@ -208,6 +210,7 @@ public class SCMNodeManager
registerMXBean();
+ this.nodePoolManager = new SCMNodePoolManager(conf);
this.scmManager = scmManager;
}
@@ -679,6 +682,7 @@ public class SCMNodeManager
@Override
public void close() throws IOException {
unregisterMXBean();
+ nodePoolManager.close();
executorService.shutdown();
try {
if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
@@ -756,6 +760,20 @@ public class SCMNodeManager
LOG.info("Leaving startup chill mode.");
}
+ // TODO: define node pool policy for non-default node pool.
+ // For now, all nodes are added to the "DefaultNodePool" upon registration
+ // if it has not been added to any node pool yet.
+ try {
+ if (nodePoolManager.getNodePool(datanodeDetails) == null) {
+ nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
+ datanodeDetails);
+ }
+ } catch (IOException e) {
+ // TODO: make sure registration failure is handled correctly.
+ return RegisteredCommand.newBuilder()
+ .setErrorCode(ErrorCode.errorNodeNotPermitted)
+ .build();
+ }
// Updating Node Report, as registration is successful
updateNodeStat(datanodeDetails.getUuid(), nodeReport);
LOG.info("Data node with ID: {} Registered.",
@@ -842,6 +860,11 @@ public class SCMNodeManager
}
@Override
+ public NodePoolManager getNodePoolManager() {
+ return nodePoolManager;
+ }
+
+ @Override
public Map<String, Integer> getNodeCount() {
Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
for(NodeState state : NodeState.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
new file mode 100644
index 0000000..faf330e
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodePoolManager.java
@@ -0,0 +1,269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_DB_CACHE_SIZE_MB;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+ .FAILED_TO_FIND_NODE_IN_POOL;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+ .FAILED_TO_LOAD_NODEPOOL;
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
+
+/**
+ * SCM node pool manager that manges node pools.
+ */
+public final class SCMNodePoolManager implements NodePoolManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SCMNodePoolManager.class);
+ private static final List<DatanodeDetails> EMPTY_NODE_LIST =
+ new ArrayList<>();
+ private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
+ public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
+
+ // DB that saves the node to node pool mapping.
+ private MetadataStore nodePoolStore;
+
+ // In-memory node pool to nodes mapping
+ private HashMap<String, Set<DatanodeDetails>> nodePools;
+
+ // Read-write lock for nodepool operations
+ private ReadWriteLock lock;
+
+ /**
+ * Construct SCMNodePoolManager class that manages node to node pool mapping.
+ * @param conf - configuration.
+ * @throws IOException
+ */
+ public SCMNodePoolManager(final OzoneConfiguration conf)
+ throws IOException {
+ final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
+ OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
+ File metaDir = getOzoneMetaDirPath(conf);
+ String scmMetaDataDir = metaDir.getPath();
+ File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
+ nodePoolStore = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setDbFile(nodePoolDBPath)
+ .setCacheSize(cacheSize * OzoneConsts.MB)
+ .build();
+ nodePools = new HashMap<>();
+ lock = new ReentrantReadWriteLock();
+ init();
+ }
+
+ /**
+ * Initialize the in-memory store based on persist store from level db.
+ * No lock is needed as init() is only invoked by constructor.
+ * @throws SCMException
+ */
+ private void init() throws SCMException {
+ try {
+ nodePoolStore.iterate(null, (key, value) -> {
+ try {
+ DatanodeDetails nodeId = DatanodeDetails.getFromProtoBuf(
+ HddsProtos.DatanodeDetailsProto.PARSER.parseFrom(key));
+ String poolName = DFSUtil.bytes2String(value);
+
+ Set<DatanodeDetails> nodePool = null;
+ if (nodePools.containsKey(poolName)) {
+ nodePool = nodePools.get(poolName);
+ } else {
+ nodePool = new HashSet<>();
+ nodePools.put(poolName, nodePool);
+ }
+ nodePool.add(nodeId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding node: {} to node pool: {}",
+ nodeId, poolName);
+ }
+ } catch (IOException e) {
+ LOG.warn("Can't add a datanode to node pool, continue next...");
+ }
+ return true;
+ });
+ } catch (IOException e) {
+ LOG.error("Loading node pool error " + e);
+ throw new SCMException("Failed to load node pool",
+ FAILED_TO_LOAD_NODEPOOL);
+ }
+ }
+
+ /**
+ * Add a datanode to a node pool.
+ * @param pool - name of the node pool.
+ * @param node - name of the datanode.
+ */
+ @Override
+ public void addNode(final String pool, final DatanodeDetails node)
+ throws IOException {
+ Preconditions.checkNotNull(pool, "pool name is null");
+ Preconditions.checkNotNull(node, "node is null");
+ lock.writeLock().lock();
+ try {
+ // add to the persistent store
+ nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
+ DFSUtil.string2Bytes(pool));
+
+ // add to the in-memory store
+ Set<DatanodeDetails> nodePool = null;
+ if (nodePools.containsKey(pool)) {
+ nodePool = nodePools.get(pool);
+ } else {
+ nodePool = new HashSet<DatanodeDetails>();
+ nodePools.put(pool, nodePool);
+ }
+ nodePool.add(node);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Remove a datanode from a node pool.
+ * @param pool - name of the node pool.
+ * @param node - datanode id.
+ * @throws SCMException
+ */
+ @Override
+ public void removeNode(final String pool, final DatanodeDetails node)
+ throws SCMException {
+ Preconditions.checkNotNull(pool, "pool name is null");
+ Preconditions.checkNotNull(node, "node is null");
+ lock.writeLock().lock();
+ try {
+ // Remove from the persistent store
+ byte[] kName = node.getProtoBufMessage().toByteArray();
+ byte[] kData = nodePoolStore.get(kName);
+ if (kData == null) {
+ throw new SCMException(String.format("Unable to find node %s from" +
+ " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
+ FAILED_TO_FIND_NODE_IN_POOL);
+ }
+ nodePoolStore.delete(kName);
+
+ // Remove from the in-memory store
+ if (nodePools.containsKey(pool)) {
+ Set<DatanodeDetails> nodePool = nodePools.get(pool);
+ nodePool.remove(node);
+ } else {
+ throw new SCMException(String.format("Unable to find node %s from" +
+ " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
+ FAILED_TO_FIND_NODE_IN_POOL);
+ }
+ } catch (IOException e) {
+ throw new SCMException("Failed to remove node " + node.toString()
+ + " from node pool " + pool, e,
+ SCMException.ResultCodes.IO_EXCEPTION);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Get all the node pools.
+ * @return all the node pools.
+ */
+ @Override
+ public List<String> getNodePools() {
+ lock.readLock().lock();
+ try {
+ if (!nodePools.isEmpty()) {
+ return nodePools.keySet().stream().collect(Collectors.toList());
+ } else {
+ return EMPTY_NODEPOOL_LIST;
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Get all datanodes of a specific node pool.
+ * @param pool - name of the node pool.
+ * @return all datanodes of the specified node pool.
+ */
+ @Override
+ public List<DatanodeDetails> getNodes(final String pool) {
+ Preconditions.checkNotNull(pool, "pool name is null");
+ if (nodePools.containsKey(pool)) {
+ return nodePools.get(pool).stream().collect(Collectors.toList());
+ } else {
+ return EMPTY_NODE_LIST;
+ }
+ }
+
+ /**
+ * Get the node pool name if the node has been added to a node pool.
+ * @param datanodeDetails - datanode ID.
+ * @return node pool name if it has been assigned.
+ * null if the node has not been assigned to any node pool yet.
+ * TODO: Put this in a in-memory map if performance is an issue.
+ */
+ @Override
+ public String getNodePool(final DatanodeDetails datanodeDetails)
+ throws SCMException {
+ Preconditions.checkNotNull(datanodeDetails, "node is null");
+ try {
+ byte[] result = nodePoolStore.get(
+ datanodeDetails.getProtoBufMessage().toByteArray());
+ return result == null ? null : DFSUtil.bytes2String(result);
+ } catch (IOException e) {
+ throw new SCMException("Failed to get node pool for node "
+ + datanodeDetails.toString(), e,
+ SCMException.ResultCodes.IO_EXCEPTION);
+ }
+ }
+
+ /**
+ * Close node pool level db store.
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ nodePoolStore.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 80b5d6e..8c59462 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -272,6 +273,11 @@ public class MockNodeManager implements NodeManager {
return new SCMNodeMetric(nodeMetricMap.get(datanodeDetails.getUuid()));
}
+ @Override
+ public NodePoolManager getNodePoolManager() {
+ return Mockito.mock(NodePoolManager.class);
+ }
+
/**
* Used for testing.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
new file mode 100644
index 0000000..8f412de
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodePoolManager.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.commons.collections.ListUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms
+ .ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms
+ .SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for SCM node pool manager.
+ */
+public class TestSCMNodePoolManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestSCMNodePoolManager.class);
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private final File testDir = PathUtils.getTestDir(
+ TestSCMNodePoolManager.class);
+
+ SCMNodePoolManager createNodePoolManager(OzoneConfiguration conf)
+ throws IOException {
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
+ testDir.getAbsolutePath());
+ conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ return new SCMNodePoolManager(conf);
+ }
+
+ /**
+ * Test default node pool.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testDefaultNodePool() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ try {
+ final String defaultPool = "DefaultPool";
+ NodePoolManager npMgr = createNodePoolManager(conf);
+
+ final int nodeCount = 4;
+ final List<DatanodeDetails> nodes = TestUtils
+ .getListOfDatanodeDetails(nodeCount);
+ assertEquals(0, npMgr.getNodePools().size());
+ for (DatanodeDetails node: nodes) {
+ npMgr.addNode(defaultPool, node);
+ }
+ List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+ assertEquals(nodeCount, nodesRetrieved.size());
+ assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+
+ DatanodeDetails nodeRemoved = nodes.remove(2);
+ npMgr.removeNode(defaultPool, nodeRemoved);
+ List<DatanodeDetails> nodesAfterRemove = npMgr.getNodes(defaultPool);
+ assertTwoDatanodeListsEqual(nodes, nodesAfterRemove);
+
+ List<DatanodeDetails> nonExistSet = npMgr.getNodes("NonExistSet");
+ assertEquals(0, nonExistSet.size());
+ } finally {
+ FileUtil.fullyDelete(testDir);
+ }
+ }
+
+
+ /**
+ * Test default node pool reload.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testDefaultNodePoolReload() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ final String defaultPool = "DefaultPool";
+ final int nodeCount = 4;
+ final List<DatanodeDetails> nodes = TestUtils
+ .getListOfDatanodeDetails(nodeCount);
+
+ try {
+ try {
+ SCMNodePoolManager npMgr = createNodePoolManager(conf);
+ assertEquals(0, npMgr.getNodePools().size());
+ for (DatanodeDetails node : nodes) {
+ npMgr.addNode(defaultPool, node);
+ }
+ List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+ assertEquals(nodeCount, nodesRetrieved.size());
+ assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+ npMgr.close();
+ } finally {
+ LOG.info("testDefaultNodePoolReload: Finish adding nodes to pool" +
+ " and close.");
+ }
+
+ // try reload with a new NodePoolManager instance
+ try {
+ SCMNodePoolManager npMgr = createNodePoolManager(conf);
+ List<DatanodeDetails> nodesRetrieved = npMgr.getNodes(defaultPool);
+ assertEquals(nodeCount, nodesRetrieved.size());
+ assertTwoDatanodeListsEqual(nodes, nodesRetrieved);
+ } finally {
+ LOG.info("testDefaultNodePoolReload: Finish reloading node pool.");
+ }
+ } finally {
+ FileUtil.fullyDelete(testDir);
+ }
+ }
+
+ /**
+ * Compare and verify that two datanode lists are equal.
+ * @param list1 - datanode list 1.
+ * @param list2 - datanode list 2.
+ */
+ private void assertTwoDatanodeListsEqual(List<DatanodeDetails> list1,
+ List<DatanodeDetails> list2) {
+ assertEquals(list1.size(), list2.size());
+ Collections.sort(list1);
+ Collections.sort(list2);
+ assertTrue(ListUtils.isEqualList(list1, list2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index 1a4dcd7..072d821 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
@@ -200,6 +201,10 @@ public class ReplicationNodeManagerMock implements NodeManager {
return null;
}
+ @Override
+ public NodePoolManager getNodePoolManager() {
+ return Mockito.mock(NodePoolManager.class);
+ }
/**
* Wait for the heartbeat is processed by NodeManager.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
new file mode 100644
index 0000000..ffcd752
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodePoolManagerMock.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.container.testutils;
+
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.NodePoolManager;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Pool Manager replication mock.
+ */
+public class ReplicationNodePoolManagerMock implements NodePoolManager {
+
+ private final Map<DatanodeDetails, String> nodeMemberShip;
+
+ /**
+ * A node pool manager for testing.
+ */
+ public ReplicationNodePoolManagerMock() {
+ nodeMemberShip = new HashMap<>();
+ }
+
+ /**
+ * Add a node to a node pool.
+ *
+ * @param pool - name of the node pool.
+ * @param node - data node.
+ */
+ @Override
+ public void addNode(String pool, DatanodeDetails node) {
+ nodeMemberShip.put(node, pool);
+ }
+
+ /**
+ * Remove a node from a node pool.
+ *
+ * @param pool - name of the node pool.
+ * @param node - data node.
+ * @throws SCMException
+ */
+ @Override
+ public void removeNode(String pool, DatanodeDetails node)
+ throws SCMException {
+ nodeMemberShip.remove(node);
+
+ }
+
+ /**
+ * Get a list of known node pools.
+ *
+ * @return a list of known node pool names or an empty list if not node pool
+ * is defined.
+ */
+ @Override
+ public List<String> getNodePools() {
+ Set<String> poolSet = new HashSet<>();
+ for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
+ poolSet.add(entry.getValue());
+ }
+ return new ArrayList<>(poolSet);
+
+ }
+
+ /**
+ * Get all nodes of a node pool given the name of the node pool.
+ *
+ * @param pool - name of the node pool.
+ * @return a list of datanode ids or an empty list if the node pool was not
+ * found.
+ */
+ @Override
+ public List<DatanodeDetails> getNodes(String pool) {
+ Set<DatanodeDetails> datanodeSet = new HashSet<>();
+ for (Map.Entry<DatanodeDetails, String> entry : nodeMemberShip.entrySet()) {
+ if (entry.getValue().equals(pool)) {
+ datanodeSet.add(entry.getKey());
+ }
+ }
+ return new ArrayList<>(datanodeSet);
+ }
+
+ /**
+ * Get the node pool name if the node has been added to a node pool.
+ *
+ * @param datanodeDetails DatanodeDetails.
+ * @return node pool name if it has been assigned. null if the node has not
+ * been assigned to any node pool yet.
+ */
+ @Override
+ public String getNodePool(DatanodeDetails datanodeDetails) {
+ return nodeMemberShip.get(datanodeDetails);
+ }
+
+ /**
+ * Closes this stream and releases any system resources associated
+ * with it. If the stream is already closed then invoking this
+ * method has no effect.
+ * <p>
+ * <p> As noted in {@link AutoCloseable#close()}, cases where the
+ * close may fail require careful attention. It is strongly advised
+ * to relinquish the underlying resources and to internally
+ * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+ * the {@code IOException}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ @Override
+ public void close() throws IOException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d6fe5f3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index b4ed2b1..4d70af8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -51,9 +51,12 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.UUID;
+import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
import static org.apache.hadoop.ozone.OzoneConsts.KB;
+import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* This class tests the CLI that transforms container into SQLite DB files.
@@ -174,6 +177,34 @@ public class TestContainerSQLCli {
}
@Test
+ public void testConvertNodepoolDB() throws Exception {
+ String dbOutPath = GenericTestUtils.getTempPath(
+ UUID.randomUUID() + "/out_sql.db");
+ String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbPath = dbRootPath + "/" + NODEPOOL_DB;
+ String[] args = {"-p", dbPath, "-o", dbOutPath};
+
+ cli.run(args);
+
+ // verify the sqlite db
+ HashMap<String, String> expectedPool = new HashMap<>();
+ for (DatanodeDetails dnid : nodeManager.getAllNodes()) {
+ expectedPool.put(dnid.getUuidString(), "DefaultNodePool");
+ }
+ Connection conn = connectDB(dbOutPath);
+ String sql = "SELECT * FROM nodePool";
+ ResultSet rs = executeQuery(conn, sql);
+ while(rs.next()) {
+ String datanodeUUID = rs.getString("datanodeUUID");
+ String poolName = rs.getString("poolName");
+ assertTrue(expectedPool.remove(datanodeUUID).equals(poolName));
+ }
+ assertEquals(0, expectedPool.size());
+
+ Files.delete(Paths.get(dbOutPath));
+ }
+
+ @Test
public void testConvertContainerDB() throws Exception {
String dbOutPath = GenericTestUtils.getTempPath(
UUID.randomUUID() + "/out_sql.db");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-8108. Added option to disable
loading existing filters to prevent security filter from initialize twice.
Contributed by Sunil Govindan
Posted by gi...@apache.org.
YARN-8108. Added option to disable loading existing filters to prevent
security filter from initialize twice.
Contributed by Sunil Govindan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b69ba0f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b69ba0f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b69ba0f3
Branch: refs/heads/HADOOP-15461
Commit: b69ba0f3307a90500aeb0c5db9e582fcda60b501
Parents: 3e58633
Author: Eric Yang <ey...@apache.org>
Authored: Tue Jun 26 17:34:57 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Tue Jun 26 17:34:57 2018 -0400
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/webapp/WebApps.java | 14 +++++++++++---
.../yarn/server/resourcemanager/ResourceManager.java | 2 +-
2 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69ba0f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 0d045f3..0e9f0a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -83,6 +83,7 @@ public class WebApps {
public String name;
public String spec;
public Map<String, String> params;
+ public boolean loadExistingFilters = true;
}
final String name;
@@ -151,12 +152,13 @@ public class WebApps {
public Builder<T> withServlet(String name, String pathSpec,
Class<? extends HttpServlet> servlet,
- Map<String, String> params) {
+ Map<String, String> params,boolean loadExistingFilters) {
ServletStruct struct = new ServletStruct();
struct.clazz = servlet;
struct.name = name;
struct.spec = pathSpec;
struct.params = params;
+ struct.loadExistingFilters = loadExistingFilters;
servlets.add(struct);
return this;
}
@@ -256,9 +258,15 @@ public class WebApps {
pathList.add("/" + wsName + "/*");
}
}
+
for (ServletStruct s : servlets) {
if (!pathList.contains(s.spec)) {
- pathList.add(s.spec);
+ // The servlet told us to not load-existing filters, but we still want
+ // to add the default authentication filter always, so add it to the
+ // pathList
+ if (!s.loadExistingFilters) {
+ pathList.add(s.spec);
+ }
}
}
if (conf == null) {
@@ -333,7 +341,7 @@ public class WebApps {
HttpServer2 server = builder.build();
for(ServletStruct struct: servlets) {
- if (struct.params != null) {
+ if (!struct.loadExistingFilters) {
server.addInternalServlet(struct.name, struct.spec,
struct.clazz, struct.params);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69ba0f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index c533111..0b7e87c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1111,7 +1111,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
"ws")
.with(conf)
.withServlet("API-Service", "/app/*",
- ServletContainer.class, params)
+ ServletContainer.class, params, false)
.withHttpSpnegoPrincipalKey(
YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org