You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2018/05/18 00:06:20 UTC
[3/5] hadoop git commit: HDFS-13560. Insufficient system resources
exist to complete the requested service for some tests on Windows.
Contributed by Anbang Hu.
HDFS-13560. Insufficient system resources exist to complete the requested service for some tests on Windows. Contributed by Anbang Hu.
(cherry picked from commit 53b807a6a8486cefe0b036f7893de9f619bd44a1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdc16b3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdc16b3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdc16b3a
Branch: refs/heads/branch-3.0
Commit: cdc16b3a2f4ce9b2454a3e4b6e0a65598ff918fc
Parents: ccd7d50
Author: Inigo Goiri <in...@apache.org>
Authored: Thu May 17 17:03:23 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu May 17 17:04:23 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/Shell.java | 15 +++++++++++++++
.../hdfs/server/datanode/TestDirectoryScanner.java | 3 ++-
.../datanode/fsdataset/impl/LazyPersistTestCase.java | 3 ++-
.../hdfs/server/namenode/TestNameNodeMXBean.java | 7 +++++--
4 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc16b3a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index bfb8183..b74964a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -1373,4 +1373,19 @@ public abstract class Shell {
return new HashSet<>(CHILD_SHELLS.keySet());
}
}
+
+ /**
+ * Static method to return the memory lock limit for datanode.
+ * @param ulimit max value at which memory locked should be capped.
+ * @return long value specifying the memory lock limit.
+ */
+ public static Long getMemlockLimit(Long ulimit) {
+ if (WINDOWS) {
+ // HDFS-13560: if ulimit is too large on Windows, Windows will complain
+ // "1450: Insufficient system resources exist to complete the requested
+ // service". Thus, cap Windows memory lock limit at Integer.MAX_VALUE.
+ return Math.min(Integer.MAX_VALUE, ulimit);
+ }
+ return ulimit;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc16b3a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
index c95c71b..f792523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode;
+import static org.apache.hadoop.util.Shell.getMemlockLimit;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
@@ -99,7 +100,7 @@ public class TestDirectoryScanner {
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
CONF.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
- Long.MAX_VALUE);
+ getMemlockLimit(Long.MAX_VALUE));
}
@Before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc16b3a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index 799d5d1..c5ce1eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -26,6 +26,7 @@ import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.fs.StorageType.DEFAULT;
import static org.apache.hadoop.fs.StorageType.RAM_DISK;
import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
+import static org.apache.hadoop.util.Shell.getMemlockLimit;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -422,7 +423,7 @@ public abstract class LazyPersistTestCase {
private StorageType[] storageTypes = null;
private int ramDiskReplicaCapacity = -1;
private long ramDiskStorageLimit = -1;
- private long maxLockedMemory = Long.MAX_VALUE;
+ private long maxLockedMemory = getMemlockLimit(Long.MAX_VALUE);
private boolean hasTransientStorage = true;
private boolean useScr = false;
private boolean useLegacyBlockReaderLocal = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdc16b3a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index e0eddc4..ce8c06b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -75,6 +75,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.util.Shell.getMemlockLimit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -103,8 +104,10 @@ public class TestNameNodeMXBean {
@Test
public void testNameNodeMXBeanInfo() throws Exception {
Configuration conf = new Configuration();
+ Long maxLockedMemory = getMemlockLimit(
+ NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
- NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
+ maxLockedMemory);
MiniDFSCluster cluster = null;
try {
@@ -256,7 +259,7 @@ public class TestNameNodeMXBean {
assertEquals(1, statusMap.get("active").size());
assertEquals(1, statusMap.get("failed").size());
assertEquals(0L, mbs.getAttribute(mxbeanName, "CacheUsed"));
- assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
+ assertEquals(maxLockedMemory *
cluster.getDataNodes().size(),
mbs.getAttribute(mxbeanName, "CacheCapacity"));
assertNull("RollingUpgradeInfo should be null when there is no rolling"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org