You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2016/08/18 20:27:03 UTC
[01/50] [abbrv] hadoop git commit: YARN-2398. TestResourceTrackerOnHA
crashes. Contributed by Ajith S.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 d2cf8b54c -> ae4db2544
YARN-2398. TestResourceTrackerOnHA crashes. Contributed by Ajith S.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/666ad0b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/666ad0b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/666ad0b6
Branch: refs/heads/HADOOP-13345
Commit: 666ad0b67b9162860fccddbd1d89c806c70b60ef
Parents: a428d4f
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Aug 11 11:27:00 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Thu Aug 11 11:27:00 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/666ad0b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index f336b0f..a8e9132 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -328,7 +328,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
private boolean waittingForFailOver() {
int maximumWaittingTime = 50;
int count = 0;
- while (!failoverTriggered.get() && count >= maximumWaittingTime) {
+ while (!failoverTriggered.get() && count <= maximumWaittingTime) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
@@ -336,7 +336,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
}
count++;
}
- if (count >= maximumWaittingTime) {
+ if (count >= maximumWaittingTime && failoverThread != null) {
return false;
}
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HADOOP-13441. Document
LdapGroupsMapping keystore password properties. Contributed by Yuanbo Liu.
Posted by cn...@apache.org.
HADOOP-13441. Document LdapGroupsMapping keystore password properties. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d892ae95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d892ae95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d892ae95
Branch: refs/heads/HADOOP-13345
Commit: d892ae9576d07d01927443b6dc6c934a6c2f317f
Parents: 8fbb57f
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Aug 11 11:57:20 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Aug 11 11:57:43 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 5 +-
.../fs/CommonConfigurationKeysPublic.java | 26 ++++++++
.../alias/AbstractJavaKeyStoreProvider.java | 4 +-
.../security/alias/CredentialProvider.java | 6 +-
.../alias/CredentialProviderFactory.java | 3 +-
.../src/main/resources/core-default.xml | 64 ++++++++++++++++++--
.../src/site/markdown/CredentialProviderAPI.md | 2 +-
.../src/site/markdown/GroupsMapping.md | 6 +-
8 files changed, 104 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 0420c6b..29733eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -78,6 +78,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -2053,7 +2054,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
*/
protected char[] getPasswordFromConfig(String name) {
char[] pass = null;
- if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, true)) {
+ if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK,
+ CommonConfigurationKeysPublic.
+ HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT)) {
String passStr = get(name);
if (passStr != null) {
pass = passStr.toCharArray();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 8781f24..e746f2b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -727,5 +727,31 @@ public class CommonConfigurationKeysPublic {
"hadoop.http.logs.enabled";
/** Defalt value for HADOOP_HTTP_LOGS_ENABLED */
public static final boolean HADOOP_HTTP_LOGS_ENABLED_DEFAULT = true;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH =
+ "hadoop.security.credential.provider.path";
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK =
+ "hadoop.security.credential.clear-text-fallback";
+ public static final boolean
+ HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK_DEFAULT = true;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY =
+ "hadoop.security.credstore.java-keystore-provider.password-file";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
index 335c198..8e4a0a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.ProviderUtils;
@@ -64,7 +65,8 @@ public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
public static final String CREDENTIAL_PASSWORD_ENV_VAR =
"HADOOP_CREDSTORE_PASSWORD";
public static final String CREDENTIAL_PASSWORD_FILE_KEY =
- "hadoop.security.credstore.java-keystore-provider.password-file";
+ CommonConfigurationKeysPublic.
+ HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY;
public static final String CREDENTIAL_PASSWORD_DEFAULT = "none";
private Path path;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
index 1a18ca1..82b91de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
* A provider of credentials or password for Hadoop applications. Provides an
@@ -35,8 +36,9 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public abstract class CredentialProvider {
- public static final String CLEAR_TEXT_FALLBACK
- = "hadoop.security.credential.clear-text-fallback";
+ public static final String CLEAR_TEXT_FALLBACK =
+ CommonConfigurationKeysPublic.
+ HADOOP_SECURITY_CREDENTIAL_CLEAR_TEXT_FALLBACK;
/**
* The combination of both the alias and the actual credential value.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
index 0c9c21f..3c3f79f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
@@ -28,6 +28,7 @@ import java.util.ServiceLoader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
* A factory to create a list of CredentialProvider based on the path given in a
@@ -38,7 +39,7 @@ import org.apache.hadoop.conf.Configuration;
@InterfaceStability.Unstable
public abstract class CredentialProviderFactory {
public static final String CREDENTIAL_PROVIDER_PATH =
- "hadoop.security.credential.provider.path";
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_CREDENTIAL_PROVIDER_PATH;
public abstract CredentialProvider createProvider(URI providerName,
Configuration conf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 0d751bb..992b464 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -241,10 +241,52 @@
<name>hadoop.security.group.mapping.ldap.ssl.keystore.password.file</name>
<value></value>
<description>
- The path to a file containing the password of the LDAP SSL keystore.
+ The path to a file containing the password of the LDAP SSL keystore. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.ssl.keystore.password is not set,
+ LDAPGroupsMapping reads password from the file.
IMPORTANT: This file should be readable only by the Unix user running
- the daemons.
+ the daemons and should be a local file.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.ssl.keystore.password</name>
+ <value></value>
+ <description>
+ The password of the LDAP SSL keystore. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.credential.clear-text-fallback</name>
+ <value>true</value>
+ <description>
+ true or false to indicate whether or not to fall back to storing credential
+ password as clear text. The default value is true. This property only works
+ when the password can't not be found from credential providers.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.credential.provider.path</name>
+ <value></value>
+ <description>
+ A comma-separated list of URLs that indicates the type and
+ location of a list of providers that should be consulted.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.credstore.java-keystore-provider.password-file</name>
+ <value></value>
+ <description>
+ The path to a file containing the custom password for all keystores
+ that may be configured in the provider path.
</description>
</property>
@@ -261,10 +303,24 @@
<name>hadoop.security.group.mapping.ldap.bind.password.file</name>
<value></value>
<description>
- The path to a file containing the password of the bind user.
+ The path to a file containing the password of the bind user. If
+ the password is not configured in credential providers and the property
+ hadoop.security.group.mapping.ldap.bind.password is not set,
+ LDAPGroupsMapping reads password from the file.
IMPORTANT: This file should be readable only by the Unix user running
- the daemons.
+ the daemons and should be a local file.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.bind.password</name>
+ <value></value>
+ <description>
+ The password of the bind user. this property name is used as an
+ alias to get the password from credential providers. If the password can
+ not be found and hadoop.security.credential.clear-text-fallback is true
+ LDAPGroupsMapping uses the value of this property for password.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
index d6e4ee7..1142372 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md
@@ -96,7 +96,7 @@ In summary, first, provision the credentials into a provider then configure the
##### Supported Features
| Feature\Component | Description | Link |
|:---- |:---- |:---|
-|LDAPGroupsMapping |LDAPGroupsMapping is used to look up the groups for a given user in LDAP. The CredentialProvider API is used to protect the LDAP bind password and those needed for SSL.|TODO|
+|LDAPGroupsMapping |LDAPGroupsMapping is used to look up the groups for a given user in LDAP. The CredentialProvider API is used to protect the LDAP bind password and those needed for SSL.|[LDAP Groups Mapping](GroupsMapping.html#LDAP_Groups_Mapping)|
|SSL Passwords |FileBasedKeyStoresFactory leverages the credential provider API in order to resolve the SSL related passwords.|TODO|
|HDFS |DFSUtil leverages Configuration.getPassword method to use the credential provider API and/or fallback to the clear text value stored in ssl-server.xml.|TODO|
|YARN |WebAppUtils uptakes the use of the credential provider API through the new method on Configuration called getPassword. This provides an alternative to storing the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility.|TODO|
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892ae95/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index f2e04b9..b6dac85 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -99,8 +99,10 @@ If the LDAP server supports POSIX group semantics (RFC-2307), Hadoop can perform
### SSL ###
To secure the connection, the implementation supports LDAP over SSL (LDAPS). SSL is enable by setting `hadoop.security.group.mapping.ldap.ssl` to `true`.
-In addition, specify the path to the keystore file for SSL connection in `hadoop.security.group.mapping.ldap.ssl.keystore` and keystore password in `hadoop.security.group.mapping.ldap.ssl.keystore.password`.
-Alternatively, store the keystore password in a file, and point `hadoop.security.group.mapping.ldap.ssl.keystore.password.file` to that file. For security purposes, this file should be readable only by the Unix user running the daemons.
+In addition, specify the path to the keystore file for SSL connection in `hadoop.security.group.mapping.ldap.ssl.keystore` and keystore password in `hadoop.security.group.mapping.ldap.ssl.keystore.password`, at the same time, make sure `hadoop.security.credential.clear-text-fallback` is true.
+Alternatively, store the keystore password in a file, and point `hadoop.security.group.mapping.ldap.ssl.keystore.password.file` to that file.
+For security purposes, this file should be readable only by the Unix user running the daemons, and for preventing recursive dependency, this file should be a local file.
+The first approach aka using `hadoop.security.group.mapping.ldap.ssl.keystore.password` is highly discouraged because it exposes the password in the configuration file.
### Low latency group mapping resolution ###
Typically, Hadoop resolves a user's group names by making two LDAP queries: the first query gets the user object, and the second query uses the user's Distinguished Name to find the groups.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-10773. BlockSender should not
synchronize on the dataset object. (Contributed by Chen Liang)
Posted by cn...@apache.org.
HDFS-10773. BlockSender should not synchronize on the dataset object. (Contributed by Chen Liang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca13e797
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca13e797
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca13e797
Branch: refs/heads/HADOOP-13345
Commit: ca13e7971d0db0705d5e36bcf03ead3cab5ab0d7
Parents: 2aa5e2c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Aug 17 16:29:08 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Aug 17 16:29:08 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca13e797/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 398935d..7c3d778 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.SocketOutputStream;
+import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.util.DataChecksum;
import org.apache.htrace.core.TraceScope;
@@ -239,7 +240,7 @@ class BlockSender implements java.io.Closeable {
final Replica replica;
final long replicaVisibleLength;
- synchronized(datanode.data) {
+ try(AutoCloseableLock lock = datanode.data.acquireDatasetLock()) {
replica = getReplica(block, datanode);
replicaVisibleLength = replica.getVisibleLength();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-5491. Fix random failure of
TestCapacityScheduler#testCSQueueBlocked (Bibin A Chundatt via Varun Saxena)
Posted by cn...@apache.org.
YARN-5491. Fix random failure of TestCapacityScheduler#testCSQueueBlocked (Bibin A Chundatt via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d677b68c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d677b68c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d677b68c
Branch: refs/heads/HADOOP-13345
Commit: d677b68c2599445fff56db4df26448a8bad0f5dd
Parents: 23c6e3c
Author: Varun Saxena <va...@apache.org>
Authored: Mon Aug 15 03:31:21 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Mon Aug 15 03:31:21 2016 +0530
----------------------------------------------------------------------
.../scheduler/capacity/TestCapacityScheduler.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d677b68c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2da7adb..0b52b86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -3635,7 +3635,7 @@ public class TestCapacityScheduler {
}
assertEquals("A Used Resource should be 2 GB", 2 * GB,
cs.getQueue("a").getUsedResources().getMemorySize());
- assertEquals("B Used Resource should be 2 GB", 13 * GB,
+ assertEquals("B Used Resource should be 13 GB", 13 * GB,
cs.getQueue("b").getUsedResources().getMemorySize());
r1 = TestUtils.createResourceRequest(
ResourceRequest.ANY, 2 * GB, 1, true, priority, recordFactory);
@@ -3659,10 +3659,14 @@ public class TestCapacityScheduler {
ContainerId containerId2 =ContainerId.newContainerId(appAttemptId2, 11);
cs.handle(new ContainerExpiredSchedulerEvent(containerId1));
+ rm.drainEvents();
+ CapacityScheduler.schedule(cs);
+
cs.handle(new ContainerExpiredSchedulerEvent(containerId2));
CapacityScheduler.schedule(cs);
rm.drainEvents();
- assertEquals("A Used Resource should be 2 GB", 4 * GB,
+
+ assertEquals("A Used Resource should be 4 GB", 4 * GB,
cs.getQueue("a").getUsedResources().getMemorySize());
assertEquals("B Used Resource should be 12 GB", 12 * GB,
cs.getQueue("b").getUsedResources().getMemorySize());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDFS-10725. Caller context should
always be constructed by a builder. (Contributed by Mingliang Liu)
Posted by cn...@apache.org.
HDFS-10725. Caller context should always be constructed by a builder. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12ad63d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12ad63d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12ad63d7
Branch: refs/heads/HADOOP-13345
Commit: 12ad63d7232ca72be9eff5680d974fc16999aac3
Parents: 5628b36
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:13:20 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:14:05 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/ipc/CallerContext.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ad63d7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index b197575..3d21bfe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -35,7 +35,7 @@ import java.util.Arrays;
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce",
"Pig", "YARN"})
@InterfaceStability.Evolving
-public class CallerContext {
+public final class CallerContext {
public static final Charset SIGNATURE_ENCODING = StandardCharsets.UTF_8;
/** The caller context.
*
@@ -54,7 +54,7 @@ public class CallerContext {
*/
private final byte[] signature;
- public CallerContext(Builder builder) {
+ private CallerContext(Builder builder) {
this.context = builder.context;
this.signature = builder.signature;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-4676. Automatic and
Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel
Zhi. (cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Posted by cn...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 22aa0ee..5a89e54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -272,6 +272,11 @@ public class MockNodes {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
};
private static RMNode buildRMNode(int rack, final Resource perNode,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 5856e59..f843261 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -709,6 +709,9 @@ public class MockRM extends ResourceManager {
public void waitForState(NodeId nodeId, NodeState finalState)
throws InterruptedException {
RMNode node = getRMContext().getRMNodes().get(nodeId);
+ if (node == null) {
+ node = getRMContext().getInactiveRMNodes().get(nodeId);
+ }
Assert.assertNotNull("node shouldn't be null", node);
int timeWaiting = 0;
while (!finalState.equals(node.getState())) {
@@ -722,11 +725,17 @@ public class MockRM extends ResourceManager {
timeWaiting += WAIT_MS_PER_LOOP;
}
- System.out.println("Node State is : " + node.getState());
+ System.out.println("Node " + nodeId + " State is : " + node.getState());
Assert.assertEquals("Node state is not correct (timedout)", finalState,
node.getState());
}
+ public void sendNodeEvent(MockNM nm, RMNodeEventType event) throws Exception {
+ RMNodeImpl node = (RMNodeImpl)
+ getRMContext().getRMNodes().get(nm.getNodeId());
+ node.handle(new RMNodeEvent(nm.getNodeId(), event));
+ }
+
public KillApplicationResponse killApp(ApplicationId appId) throws Exception {
ApplicationClientProtocol client = getClientRMService();
KillApplicationRequest req = KillApplicationRequest.newInstance(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
new file mode 100644
index 0000000..690de30
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.DecommissioningNodesWatcher.DecommissioningNodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests DecommissioningNodesWatcher.
+ */
+public class TestDecommissioningNodesWatcher {
+ private MockRM rm;
+
+ @Test
+ public void testDecommissioningNodesWatcher() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT, "40");
+
+ rm = new MockRM(conf);
+ rm.start();
+
+ DecommissioningNodesWatcher watcher =
+ new DecommissioningNodesWatcher(rm.getRMContext());
+
+ MockNM nm1 = rm.registerNode("host1:1234", 10240);
+ RMNode node1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ NodeId id1 = nm1.getNodeId();
+
+ rm.waitForState(id1, NodeState.RUNNING);
+ Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1));
+
+ RMApp app = rm.submitApp(2000);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+
+ // Setup nm1 as DECOMMISSIONING for DecommissioningNodesWatcher.
+ rm.sendNodeEvent(nm1, RMNodeEventType.GRACEFUL_DECOMMISSION);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+
+ // Update status with decreasing number of running containers until 0.
+ watcher.update(node1, createNodeStatus(id1, app, 12));
+ watcher.update(node1, createNodeStatus(id1, app, 11));
+ Assert.assertFalse(watcher.checkReadyToBeDecommissioned(id1));
+
+ watcher.update(node1, createNodeStatus(id1, app, 1));
+ Assert.assertEquals(DecommissioningNodeStatus.WAIT_CONTAINER,
+ watcher.checkDecommissioningStatus(id1));
+
+ watcher.update(node1, createNodeStatus(id1, app, 0));
+ Assert.assertEquals(DecommissioningNodeStatus.WAIT_APP,
+ watcher.checkDecommissioningStatus(id1));
+
+ // Set app to be FINISHED and verified DecommissioningNodeStatus is READY.
+ MockRM.finishAMAndVerifyAppState(app, rm, nm1, am);
+ rm.waitForState(app.getApplicationId(), RMAppState.FINISHED);
+ Assert.assertEquals(DecommissioningNodeStatus.READY,
+ watcher.checkDecommissioningStatus(id1));
+ }
+
+ @After
+ public void tearDown() {
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+
+ private NodeStatus createNodeStatus(
+ NodeId nodeId, RMApp app, int numRunningContainers) {
+ return NodeStatus.newInstance(
+ nodeId, 0, getContainerStatuses(app, numRunningContainers),
+ new ArrayList<ApplicationId>(),
+ NodeHealthStatus.newInstance(
+ true, "", System.currentTimeMillis() - 1000),
+ null, null, null);
+ }
+
+ // Get mocked ContainerStatus for bunch of containers,
+ // where numRunningContainers are RUNNING.
+ private List<ContainerStatus> getContainerStatuses(
+ RMApp app, int numRunningContainers) {
+ // Total 12 containers
+ final int total = 12;
+ numRunningContainers = Math.min(total, numRunningContainers);
+ List<ContainerStatus> output = new ArrayList<ContainerStatus>();
+ for (int i = 0; i < total; i++) {
+ ContainerState cstate = (i >= numRunningContainers)?
+ ContainerState.COMPLETE : ContainerState.RUNNING;
+ output.add(ContainerStatus.newInstance(
+ ContainerId.newContainerId(
+ ApplicationAttemptId.newInstance(app.getApplicationId(), i), 1),
+ cstate, "Dummy", 0));
+ }
+ return output;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 83a7c73..e82b93c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -254,17 +254,6 @@ public class TestRMNodeTransitions {
cm.getNumDecommissioningNMs());
Assert.assertEquals("Decommissioned Nodes", initialDecommissioned,
cm.getNumDecommisionedNMs());
-
- // Verify node in DECOMMISSIONING will be changed by status update
- // without running apps
- statusEvent = getMockRMNodeStatusEventWithoutRunningApps();
- node.handle(statusEvent);
- Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
- Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
- Assert.assertEquals("Decommissioning Nodes", initialDecommissioning - 1,
- cm.getNumDecommissioningNMs());
- Assert.assertEquals("Decommissioned Nodes", initialDecommissioned + 1,
- cm.getNumDecommisionedNMs());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 098ba54..aa5b336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
@@ -87,6 +88,9 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
private final static File TEMP_DIR = new File(System.getProperty(
"test.build.data", "/tmp"), "decommision");
private final File hostFile = new File(TEMP_DIR + File.separator + "hostFile.txt");
+ private final File excludeHostFile = new File(TEMP_DIR + File.separator +
+ "excludeHostFile.txt");
+
private MockRM rm;
/**
@@ -222,6 +226,109 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
/**
+ * Graceful decommission node with no running application.
+ */
+ @Test
+ public void testGracefulDecommissionNoApp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+
+ writeToHostsFile("");
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ MockNM nm3 = rm.registerNode("host3:4433", 5120);
+
+ int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs();
+ NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat3 = nm3.nodeHeartbeat(true);
+
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat1.getNodeAction()));
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat2.getNodeAction()));
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat3.getNodeAction()));
+
+ rm.waitForState(nm2.getNodeId(), NodeState.RUNNING);
+ rm.waitForState(nm3.getNodeId(), NodeState.RUNNING);
+
+ // Graceful decommission both host2 and host3.
+ writeToHostsFile("host2", "host3");
+ rm.getNodesListManager().refreshNodes(conf, true);
+
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONING);
+ rm.waitForState(nm3.getNodeId(), NodeState.DECOMMISSIONING);
+
+ nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ nodeHeartbeat3 = nm3.nodeHeartbeat(true);
+
+ checkDecommissionedNMCount(rm, metricCount + 2);
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONED);
+ rm.waitForState(nm3.getNodeId(), NodeState.DECOMMISSIONED);
+
+ Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat1.getNodeAction()));
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat2.getNodeAction());
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat3.getNodeAction());
+ }
+
+ /**
+ * Graceful decommission node with running application.
+ */
+ @Test
+ public void testGracefulDecommissionWithApp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+
+ writeToHostsFile("");
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 10240);
+ MockNM nm2 = rm.registerNode("host2:5678", 20480);
+ MockNM nm3 = rm.registerNode("host3:4433", 10240);
+ NodeId id1 = nm1.getNodeId();
+ NodeId id3 = nm3.getNodeId();
+ rm.waitForState(id1, NodeState.RUNNING);
+ rm.waitForState(id3, NodeState.RUNNING);
+
+ // Create an app and launch two containers on host1.
+ RMApp app = rm.submitApp(2000);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId();
+
+ // Graceful decommission host1 and host3
+ writeToHostsFile("host1", "host3");
+ rm.getNodesListManager().refreshNodes(conf, true);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+ rm.waitForState(id3, NodeState.DECOMMISSIONING);
+
+ // host1 should be DECOMMISSIONING due to running containers.
+ // host3 should become DECOMMISSIONED.
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm3.nodeHeartbeat(true);
+ rm.waitForState(id1, NodeState.DECOMMISSIONING);
+ rm.waitForState(id3, NodeState.DECOMMISSIONED);
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+
+ // Complete containers on host1.
+ // Since the app is still RUNNING, expect NodeAction.NORMAL.
+ NodeHeartbeatResponse nodeHeartbeat1 =
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.COMPLETE);
+ Assert.assertEquals(NodeAction.NORMAL, nodeHeartbeat1.getNodeAction());
+
+ // Finish the app and verified DECOMMISSIONED.
+ MockRM.finishAMAndVerifyAppState(app, rm, nm1, am);
+ rm.waitForState(app.getApplicationId(), RMAppState.FINISHED);
+ nodeHeartbeat1 = nm1.nodeHeartbeat(aaid, 2, ContainerState.COMPLETE);
+ Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat1.getNodeAction());
+ rm.waitForState(id1, NodeState.DECOMMISSIONED);
+ }
+
+ /**
* Decommissioning using a post-configured include hosts file
*/
@Test
@@ -1225,19 +1332,17 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm1 = new MockNM("host1:1234", 5120, resourceTrackerService);
RegisterNodeManagerResponse response = nm1.registerNode();
Assert.assertEquals(NodeAction.NORMAL, response.getNodeAction());
+ int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
writeToHostsFile("host2");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
rm.getNodesListManager().refreshNodes(conf);
NodeHeartbeatResponse heartbeatResponse = nm1.nodeHeartbeat(true);
Assert.assertEquals(NodeAction.SHUTDOWN, heartbeatResponse.getNodeAction());
- int shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
- checkShutdownNMCount(rm, shutdownNMsCount);
checkDecommissionedNMCount(rm, decommisionedNMsCount);
request.setNodeId(nm1.getNodeId());
resourceTrackerService.unRegisterNodeManager(request);
- shutdownNMsCount = ClusterMetrics.getMetrics().getNumShutdownNMs();
- checkShutdownNMCount(rm, shutdownNMsCount);
+ checkShutdownNMCount(rm, ++shutdownNMsCount);
checkDecommissionedNMCount(rm, decommisionedNMsCount);
// 1. Register the Node Manager
@@ -1273,8 +1378,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host1");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1300,8 +1403,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals("The inactiveRMNodes should contain an entry for the" +
"decommissioned node",
1, rm1.getRMContext().getInactiveRMNodes().size());
- excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1331,8 +1432,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
//host3 will not register or heartbeat
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host3", "host2");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
@@ -1364,14 +1463,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
MockNM nm2 = rm.registerNode("host2:5678", 10240);
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
- File excludeHostFile =
- new File(TEMP_DIR + File.separator + "excludeHostFile.txt");
writeToHostsFile(excludeHostFile, "host3", "host2");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
excludeHostFile.getAbsolutePath());
writeToHostsFile(hostFile, "host1", "host2");
writeToHostsFile(excludeHostFile, "host1");
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
rm.drainEvents();
nm1.nodeHeartbeat(true);
rm.drainEvents();
@@ -1380,7 +1477,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
.getInactiveRMNodes().get(nm1.getNodeId()).getState() == NodeState
.DECOMMISSIONED);
writeToHostsFile(excludeHostFile, "");
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
rm.drainEvents();
Assert.assertTrue("Node " + nm1.getNodeId().getHost() +
" should be Decommissioned", rm.getRMContext()
@@ -1390,7 +1487,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
/**
- * Remove a node from all lists and check if its forgotten
+ * Remove a node from all lists and check if its forgotten.
*/
@Test
public void testNodeRemovalNormally() throws Exception {
@@ -1411,7 +1508,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
public void refreshNodesOption(boolean doGraceful, Configuration conf)
throws Exception {
if (doGraceful) {
- rm.getNodesListManager().refreshNodesGracefully(conf);
+ rm.getNodesListManager().refreshNodesGracefully(conf, null);
} else {
rm.getNodesListManager().refreshNodes(conf);
}
@@ -1420,8 +1517,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
public void testNodeRemovalUtil(boolean doGraceful) throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, "");
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, "");
conf.setInt(YarnConfiguration.RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC,
@@ -1455,18 +1550,23 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
.getAbsolutePath());
refreshNodesOption(doGraceful, conf);
+ if (doGraceful) {
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONING);
+ }
nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
rm.drainEvents();
Assert.assertTrue("Node should not be in active node list",
!rmContext.getRMNodes().containsKey(nm2.getNodeId()));
RMNode rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertEquals("Node should be in inactive node list",
- rmNode.getState(), NodeState.SHUTDOWN);
+ rmNode.getState(),
+ doGraceful? NodeState.DECOMMISSIONED : NodeState.SHUTDOWN);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
- Assert.assertEquals("Shutdown nodes should be 1",
- metrics.getNumShutdownNMs(), 1);
+ Assert.assertEquals("Shutdown nodes should be expected",
+ metrics.getNumShutdownNMs(), doGraceful? 0 : 1);
int nodeRemovalTimeout =
conf.getInt(
@@ -1491,14 +1591,18 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.drainEvents();
writeToHostsFile("host1", ip);
refreshNodesOption(doGraceful, conf);
+ rm.waitForState(nm2.getNodeId(),
+ doGraceful? NodeState.DECOMMISSIONING : NodeState.SHUTDOWN);
+ nm2.nodeHeartbeat(true);
rm.drainEvents();
rmNode = rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertEquals("Node should be shutdown",
- rmNode.getState(), NodeState.SHUTDOWN);
+ rmNode.getState(),
+ doGraceful? NodeState.DECOMMISSIONED : NodeState.SHUTDOWN);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
- Assert.assertEquals("Shutdown nodes should be 1",
- metrics.getNumShutdownNMs(), 1);
+ Assert.assertEquals("Shutdown nodes should be expected",
+ metrics.getNumShutdownNMs(), doGraceful? 0 : 1);
//add back the node before timer expires
latch.await(maxThreadSleeptime - 2000, TimeUnit.MILLISECONDS);
@@ -1542,6 +1646,20 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
//Test decommed/ing node that transitions to untracked,timer should remove
+ testNodeRemovalUtilDecomToUntracked(rmContext, conf, nm1, nm2, nm3,
+ maxThreadSleeptime, doGraceful);
+ rm.stop();
+ }
+
+ // A helper method used by testNodeRemovalUtil to avoid exceeding
+ // max allowed length.
+ private void testNodeRemovalUtilDecomToUntracked(
+ RMContext rmContext, Configuration conf,
+ MockNM nm1, MockNM nm2, MockNM nm3,
+ long maxThreadSleeptime, boolean doGraceful) throws Exception {
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ String ip = NetUtils.normalizeHostName("localhost");
+ CountDownLatch latch = new CountDownLatch(1);
writeToHostsFile("host1", ip, "host2");
writeToHostsFile(excludeHostFile, "host2");
refreshNodesOption(doGraceful, conf);
@@ -1549,7 +1667,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
//nm2.nodeHeartbeat(true);
nm3.nodeHeartbeat(true);
latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
- rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
+ RMNode rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
rmContext.getInactiveRMNodes().get(nm2.getNodeId());
Assert.assertNotEquals("Timer for this node was not canceled!",
rmNode, null);
@@ -1560,6 +1678,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
writeToHostsFile("host1", ip);
writeToHostsFile(excludeHostFile, "");
refreshNodesOption(doGraceful, conf);
+ nm2.nodeHeartbeat(true);
latch.await(maxThreadSleeptime, TimeUnit.MILLISECONDS);
rmNode = doGraceful ? rmContext.getRMNodes().get(nm2.getNodeId()) :
rmContext.getInactiveRMNodes().get(nm2.getNodeId());
@@ -1571,16 +1690,12 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
metrics.getNumShutdownNMs(), 0);
Assert.assertEquals("Active nodes should be 2",
metrics.getNumActiveNMs(), 2);
-
- rm.stop();
}
private void testNodeRemovalUtilLost(boolean doGraceful) throws Exception {
Configuration conf = new Configuration();
conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 2000);
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1613,7 +1728,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals("All 3 nodes should be active",
metrics.getNumActiveNMs(), 3);
int waitCount = 0;
- while(waitCount ++<20){
+ while(waitCount++ < 20){
synchronized (this) {
wait(200);
}
@@ -1665,8 +1780,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1737,8 +1850,6 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
throws Exception {
Configuration conf = new Configuration();
int timeoutValue = 500;
- File excludeHostFile = new File(TEMP_DIR + File.separator +
- "excludeHostFile.txt");
conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
hostFile.getAbsolutePath());
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
@@ -1782,15 +1893,19 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
nm2.nodeHeartbeat(false);
nm3.nodeHeartbeat(true);
rm.drainEvents();
- Assert.assertNotEquals("host2 should be a shutdown NM!",
- rmContext.getInactiveRMNodes().get(nm2.getNodeId()), null);
- Assert.assertEquals("host2 should be a shutdown NM!",
- rmContext.getInactiveRMNodes().get(nm2.getNodeId()).getState(),
- NodeState.SHUTDOWN);
+ if (!doGraceful) {
+ Assert.assertNotEquals("host2 should be a shutdown NM!",
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId()), null);
+ Assert.assertEquals("host2 should be a shutdown NM!",
+ rmContext.getInactiveRMNodes().get(nm2.getNodeId()).getState(),
+ NodeState.SHUTDOWN);
+ }
Assert.assertEquals("There should be 2 Active NM!",
clusterMetrics.getNumActiveNMs(), 2);
- Assert.assertEquals("There should be 1 Shutdown NM!",
- clusterMetrics.getNumShutdownNMs(), 1);
+ if (!doGraceful) {
+ Assert.assertEquals("There should be 1 Shutdown NM!",
+ clusterMetrics.getNumShutdownNMs(), 1);
+ }
Assert.assertEquals("There should be 0 Unhealthy NM!",
clusterMetrics.getUnhealthyNMs(), 0);
int nodeRemovalTimeout =
@@ -1818,7 +1933,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
private void writeToHostsFile(String... hosts) throws IOException {
- writeToHostsFile(hostFile, hosts);
+ writeToHostsFile(hostFile, hosts);
}
private void writeToHostsFile(File file, String... hosts)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
index 2c926d9..e7c7e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
@@ -210,8 +210,6 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase {
nm1.registerNode();
rm.waitForState(nm1.getNodeId(), NodeState.RUNNING);
- rm.getRMContext().getNodesListManager().getHostsReader().
- getExcludedHosts().add("127.0.0.1");
rm.getRMContext().getDispatcher().getEventHandler().handle(
new RMNodeEvent(nm1.getNodeId(),
RMNodeEventType.GRACEFUL_DECOMMISSION));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 40704f0..2c38967 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -243,7 +243,7 @@ Usage:
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -refreshQueues | Reload the queues' acls, states and scheduler specific properties. ResourceManager will reload the mapred-queues configuration file. |
-| -refreshNodes | Refresh the hosts information at the ResourceManager. |
+| -refreshNodes [-g|graceful [timeout in seconds] -client|server] | Refresh the hosts information at the ResourceManager. -g option indicates graceful decommission of excluded hosts, in which case, the optional timeout indicates maximal time in seconds ResourceManager should wait before forcefully mark the node as decommissioned. |
| -refreshNodesResources | Refresh resources of NodeManagers at the ResourceManager. |
| -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings. |
| -refreshUserToGroupsMappings | Refresh user-to-groups mappings. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-13437. KMS should reload
whitelist and default key ACLs when hot-reloading. Contributed by Xiao Chen.
Posted by cn...@apache.org.
HADOOP-13437. KMS should reload whitelist and default key ACLs when hot-reloading. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9daa9979
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9daa9979
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9daa9979
Branch: refs/heads/HADOOP-13345
Commit: 9daa9979a1f92fb3230361c10ddfcc1633795c0e
Parents: 864f878
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Aug 15 18:13:58 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Aug 15 18:14:45 2016 -0700
----------------------------------------------------------------------
.../hadoop/crypto/key/kms/server/KMSACLs.java | 75 ++++----
.../crypto/key/kms/server/TestKMSACLs.java | 174 ++++++++++++++++++-
2 files changed, 207 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9daa9979/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
index 5b67950..c36fcf8 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java
@@ -34,7 +34,6 @@ import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
@@ -74,10 +73,10 @@ public class KMSACLs implements Runnable, KeyACLs {
private volatile Map<Type, AccessControlList> blacklistedAcls;
@VisibleForTesting
volatile Map<String, HashMap<KeyOpType, AccessControlList>> keyAcls;
- private final Map<KeyOpType, AccessControlList> defaultKeyAcls =
- new HashMap<KeyOpType, AccessControlList>();
- private final Map<KeyOpType, AccessControlList> whitelistKeyAcls =
- new HashMap<KeyOpType, AccessControlList>();
+ @VisibleForTesting
+ volatile Map<KeyOpType, AccessControlList> defaultKeyAcls = new HashMap<>();
+ @VisibleForTesting
+ volatile Map<KeyOpType, AccessControlList> whitelistKeyAcls = new HashMap<>();
private ScheduledExecutorService executorService;
private long lastReload;
@@ -111,7 +110,8 @@ public class KMSACLs implements Runnable, KeyACLs {
blacklistedAcls = tempBlacklist;
}
- private void setKeyACLs(Configuration conf) {
+ @VisibleForTesting
+ void setKeyACLs(Configuration conf) {
Map<String, HashMap<KeyOpType, AccessControlList>> tempKeyAcls =
new HashMap<String, HashMap<KeyOpType,AccessControlList>>();
Map<String, String> allKeyACLS =
@@ -148,38 +148,43 @@ public class KMSACLs implements Runnable, KeyACLs {
}
}
}
-
keyAcls = tempKeyAcls;
+
+ final Map<KeyOpType, AccessControlList> tempDefaults = new HashMap<>();
+ final Map<KeyOpType, AccessControlList> tempWhitelists = new HashMap<>();
for (KeyOpType keyOp : KeyOpType.values()) {
- if (!defaultKeyAcls.containsKey(keyOp)) {
- String confKey = KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + keyOp;
- String aclStr = conf.get(confKey);
- if (aclStr != null) {
- if (keyOp == KeyOpType.ALL) {
- // Ignore All operation for default key acl
- LOG.warn("Should not configure default key ACL for KEY_OP '{}'", keyOp);
- } else {
- if (aclStr.equals("*")) {
- LOG.info("Default Key ACL for KEY_OP '{}' is set to '*'", keyOp);
- }
- defaultKeyAcls.put(keyOp, new AccessControlList(aclStr));
- }
- }
- }
- if (!whitelistKeyAcls.containsKey(keyOp)) {
- String confKey = KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + keyOp;
- String aclStr = conf.get(confKey);
- if (aclStr != null) {
- if (keyOp == KeyOpType.ALL) {
- // Ignore All operation for whitelist key acl
- LOG.warn("Should not configure whitelist key ACL for KEY_OP '{}'", keyOp);
- } else {
- if (aclStr.equals("*")) {
- LOG.info("Whitelist Key ACL for KEY_OP '{}' is set to '*'", keyOp);
- }
- whitelistKeyAcls.put(keyOp, new AccessControlList(aclStr));
- }
+ parseAclsWithPrefix(conf, KMSConfiguration.DEFAULT_KEY_ACL_PREFIX,
+ keyOp, tempDefaults);
+ parseAclsWithPrefix(conf, KMSConfiguration.WHITELIST_KEY_ACL_PREFIX,
+ keyOp, tempWhitelists);
+ }
+ defaultKeyAcls = tempDefaults;
+ whitelistKeyAcls = tempWhitelists;
+ }
+
+ /**
+ * Parse the acls from configuration with the specified prefix. Currently
+ * only 2 possible prefixes: whitelist and default.
+ *
+ * @param conf The configuration.
+ * @param prefix The prefix.
+ * @param keyOp The key operation.
+ * @param results The collection of results to add to.
+ */
+ private void parseAclsWithPrefix(final Configuration conf,
+ final String prefix, final KeyOpType keyOp,
+ Map<KeyOpType, AccessControlList> results) {
+ String confKey = prefix + keyOp;
+ String aclStr = conf.get(confKey);
+ if (aclStr != null) {
+ if (keyOp == KeyOpType.ALL) {
+ // Ignore All operation for default key and whitelist key acls
+ LOG.warn("Invalid KEY_OP '{}' for {}, ignoring", keyOp, prefix);
+ } else {
+ if (aclStr.equals("*")) {
+ LOG.info("{} for KEY_OP '{}' is set to '*'", prefix, keyOp);
}
+ results.put(keyOp, new AccessControlList(aclStr));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9daa9979/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
index b4bf504..4828fe1 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSACLs.java
@@ -17,12 +17,25 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import static org.apache.hadoop.crypto.key.kms.server.KMSConfiguration.*;
+import static org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KEY_ACL;
+import static org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider.KeyOpType;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.rules.Timeout;
import org.junit.Test;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+
public class TestKMSACLs {
+ @Rule
+ public final Timeout globalTimeout = new Timeout(180000);
@Test
public void testDefaults() {
@@ -51,13 +64,160 @@ public class TestKMSACLs {
@Test
public void testKeyAclConfigurationLoad() {
final Configuration conf = new Configuration(false);
- conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_1.MANAGEMENT", "CREATE");
- conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_2.ALL", "CREATE");
- conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "test_key_3.NONEXISTOPERATION", "CREATE");
- conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "ROLLOVER");
- conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT", "DECRYPT_EEK");
+ conf.set(KEY_ACL + "test_key_1.MANAGEMENT", "CREATE");
+ conf.set(KEY_ACL + "test_key_2.ALL", "CREATE");
+ conf.set(KEY_ACL + "test_key_3.NONEXISTOPERATION", "CREATE");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "ROLLOVER");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT", "DECRYPT_EEK");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "ALL", "invalid");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "ALL", "invalid");
+ final KMSACLs acls = new KMSACLs(conf);
+ Assert.assertTrue("expected key ACL size is 2 but got "
+ + acls.keyAcls.size(), acls.keyAcls.size() == 2);
+ Assert.assertTrue("expected whitelist ACL size is 1 but got "
+ + acls.whitelistKeyAcls.size(), acls.whitelistKeyAcls.size() == 1);
+ Assert.assertFalse("ALL should not be allowed for whitelist ACLs.",
+ acls.whitelistKeyAcls.containsKey(KeyOpType.ALL));
+ Assert.assertTrue("expected default ACL size is 1 but got "
+ + acls.defaultKeyAcls.size(), acls.defaultKeyAcls.size() == 1);
+ Assert.assertTrue("ALL should not be allowed for default ACLs.",
+ acls.defaultKeyAcls.size() == 1);
+ }
+
+ @Test
+ public void testKeyAclDuplicateEntries() {
+ final Configuration conf = new Configuration(false);
+ conf.set(KEY_ACL + "test_key_1.DECRYPT_EEK", "decrypt1");
+ conf.set(KEY_ACL + "test_key_2.ALL", "all2");
+ conf.set(KEY_ACL + "test_key_1.DECRYPT_EEK", "decrypt2");
+ conf.set(KEY_ACL + "test_key_2.ALL", "all1,all3");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "default1");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "*");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK", "whitelist1");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK", "*");
final KMSACLs acls = new KMSACLs(conf);
- Assert.assertTrue("expected key ACL size is 2 but got " + acls.keyAcls.size(),
- acls.keyAcls.size() == 2);
+ Assert.assertTrue("expected key ACL size is 2 but got "
+ + acls.keyAcls.size(), acls.keyAcls.size() == 2);
+ assertKeyAcl("test_key_1", acls, KeyOpType.DECRYPT_EEK, "decrypt2");
+ assertKeyAcl("test_key_2", acls, KeyOpType.ALL, "all1", "all3");
+ assertDefaultKeyAcl(acls, KeyOpType.MANAGEMENT);
+ assertDefaultKeyAcl(acls, KeyOpType.DECRYPT_EEK);
+ AccessControlList acl = acls.whitelistKeyAcls.get(KeyOpType.DECRYPT_EEK);
+ Assert.assertNotNull(acl);
+ Assert.assertTrue(acl.isAllAllowed());
+ }
+
+ @Test
+ public void testKeyAclReload() {
+ Configuration conf = new Configuration(false);
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "READ", "read1");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK", "*");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "decrypt1");
+ conf.set(KEY_ACL + "testuser1.ALL", "testkey1");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "READ", "admin_read1");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT", "");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK", "*");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK", "admin_decrypt1");
+ final KMSACLs acls = new KMSACLs(conf);
+
+ // update config and hot-reload.
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "READ", "read2");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT", "mgmt1,mgmt2");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK", "");
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "decrypt2");
+ conf.set(KEY_ACL + "testkey1.ALL", "testkey1,testkey2");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "READ", "admin_read2");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT", "admin_mgmt,admin_mgmt1");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK", "");
+ conf.set(WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK", "admin_decrypt2");
+ acls.setKeyACLs(conf);
+
+ assertDefaultKeyAcl(acls, KeyOpType.READ, "read2");
+ assertDefaultKeyAcl(acls, KeyOpType.MANAGEMENT, "mgmt1", "mgmt2");
+ assertDefaultKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertDefaultKeyAcl(acls, KeyOpType.DECRYPT_EEK, "decrypt2");
+ assertKeyAcl("testuser1", acls, KeyOpType.ALL, "testkey1");
+ assertWhitelistKeyAcl(acls, KeyOpType.READ, "admin_read2");
+ assertWhitelistKeyAcl(acls, KeyOpType.MANAGEMENT,
+ "admin_mgmt", "admin_mgmt1");
+ assertWhitelistKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertWhitelistKeyAcl(acls, KeyOpType.DECRYPT_EEK, "admin_decrypt2");
+
+ // reloading same config, nothing should change.
+ acls.setKeyACLs(conf);
+ assertDefaultKeyAcl(acls, KeyOpType.READ, "read2");
+ assertDefaultKeyAcl(acls, KeyOpType.MANAGEMENT, "mgmt1", "mgmt2");
+ assertDefaultKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertDefaultKeyAcl(acls, KeyOpType.DECRYPT_EEK, "decrypt2");
+ assertKeyAcl("testuser1", acls, KeyOpType.ALL, "testkey1");
+ assertWhitelistKeyAcl(acls, KeyOpType.READ, "admin_read2");
+ assertWhitelistKeyAcl(acls, KeyOpType.MANAGEMENT,
+ "admin_mgmt", "admin_mgmt1");
+ assertWhitelistKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertWhitelistKeyAcl(acls, KeyOpType.DECRYPT_EEK, "admin_decrypt2");
+
+ // test wildcard.
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "*");
+ acls.setKeyACLs(conf);
+ AccessControlList acl = acls.defaultKeyAcls.get(KeyOpType.DECRYPT_EEK);
+ Assert.assertTrue(acl.isAllAllowed());
+ Assert.assertTrue(acl.getUsers().isEmpty());
+ // everything else should still be the same.
+ assertDefaultKeyAcl(acls, KeyOpType.READ, "read2");
+ assertDefaultKeyAcl(acls, KeyOpType.MANAGEMENT, "mgmt1", "mgmt2");
+ assertDefaultKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertKeyAcl("testuser1", acls, KeyOpType.ALL, "testkey1");
+ assertWhitelistKeyAcl(acls, KeyOpType.READ, "admin_read2");
+ assertWhitelistKeyAcl(acls, KeyOpType.MANAGEMENT,
+ "admin_mgmt", "admin_mgmt1");
+ assertWhitelistKeyAcl(acls, KeyOpType.GENERATE_EEK);
+ assertWhitelistKeyAcl(acls, KeyOpType.DECRYPT_EEK, "admin_decrypt2");
+
+ // test new configuration should clear other items
+ conf = new Configuration();
+ conf.set(DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK", "new");
+ acls.setKeyACLs(conf);
+ assertDefaultKeyAcl(acls, KeyOpType.DECRYPT_EEK, "new");
+ Assert.assertTrue(acls.keyAcls.isEmpty());
+ Assert.assertTrue(acls.whitelistKeyAcls.isEmpty());
+ Assert.assertEquals("Got unexpected sized acls:"
+ + acls.defaultKeyAcls, 1, acls.defaultKeyAcls.size());
+ }
+
+ private void assertDefaultKeyAcl(final KMSACLs acls, final KeyOpType op,
+ final String... names) {
+ final AccessControlList acl = acls.defaultKeyAcls.get(op);
+ assertAcl(acl, op, names);
+ }
+
+ private void assertWhitelistKeyAcl(final KMSACLs acls, final KeyOpType op,
+ final String... names) {
+ final AccessControlList acl = acls.whitelistKeyAcls.get(op);
+ assertAcl(acl, op, names);
+ }
+
+ private void assertKeyAcl(final String keyName, final KMSACLs acls,
+ final KeyOpType op, final String... names) {
+ Assert.assertTrue(acls.keyAcls.containsKey(keyName));
+ final HashMap<KeyOpType, AccessControlList> keyacl =
+ acls.keyAcls.get(keyName);
+ Assert.assertNotNull(keyacl.get(op));
+ assertAcl(keyacl.get(op), op, names);
+ }
+
+ private void assertAcl(final AccessControlList acl,
+ final KeyOpType op, final String... names) {
+ Assert.assertNotNull(acl);
+ Assert.assertFalse(acl.isAllAllowed());
+ final Collection<String> actual = acl.getUsers();
+ final HashSet<String> expected = new HashSet<>();
+ for (String name : names) {
+ expected.add(name);
+ }
+ Assert.assertEquals("defaultKeyAcls don't match for op:" + op,
+ expected, actual);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-5475. Fix test failure of
TestAggregatedLogFormat#testReadAcontainerLogs1 (Jun Gong via Varun Saxena)
Posted by cn...@apache.org.
YARN-5475. Fix test failure of TestAggregatedLogFormat#testReadAcontainerLogs1 (Jun Gong via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b427ce12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b427ce12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b427ce12
Branch: refs/heads/HADOOP-13345
Commit: b427ce12bcdd35e12b212c18430f0114fbbc1fea
Parents: ffe1fff
Author: Varun Saxena <va...@apache.org>
Authored: Tue Aug 16 20:24:53 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 16 20:24:53 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b427ce12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index 45dd8ab..8cbec10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -140,7 +140,8 @@ public class TestAggregatedLogFormat {
final int ch = filler;
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);
+ LogWriter logWriter = new LogWriter(new Configuration(), remoteAppLogFile,
+ ugi);
LogKey logKey = new LogKey(testContainerId);
LogValue logValue =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-13324. s3a tests don't
authenticate with S3 frankfurt (or other V4 auth only endpoints). Contributed
by Steve Loughran.
Posted by cn...@apache.org.
HADOOP-13324. s3a tests don't authenticate with S3 frankfurt (or other V4 auth only endpoints). Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3808876c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3808876c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3808876c
Branch: refs/heads/HADOOP-13345
Commit: 3808876c7397ea68906bc5cc18fdf690c9c42131
Parents: 27a6e09
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Aug 16 17:05:52 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Aug 16 17:05:52 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/tools/hadoop-aws/index.md | 247 ++++++++++++++++---
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 15 ++
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 1 +
.../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 15 +-
.../scale/TestS3AInputStreamPerformance.java | 2 +
5 files changed, 247 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808876c/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 01a2bae..cb1df83 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -634,6 +634,60 @@ this capability.
any call to setReadahead() is made to an open stream.</description>
</property>
+### Working with buckets in different regions
+
+S3 Buckets are hosted in different regions, the default being US-East.
+The client talks to it by default, under the URL `s3.amazonaws.com`
+
+S3A can work with buckets from any region. Each region has its own
+S3 endpoint, documented [by Amazon](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region).
+
+1. Applications running in EC2 infrastructure do not pay for IO to/from
+*local S3 buckets*. They will be billed for access to remote buckets. Always
+use local buckets and local copies of data, wherever possible.
+1. The default S3 endpoint can support data IO with any bucket when the V1 request
+signing protocol is used.
+1. When the V4 signing protocol is used, AWS requires the explicit region endpoint
+to be used \u2014hence S3A must be configured to use the specific endpoint. This
+is done in the configuration option `fs.s3a.endpoint`.
+1. All endpoints other than the default endpoint only support interaction
+with buckets local to that S3 instance.
+
+While it is generally simpler to use the default endpoint, working with
+V4-signing-only regions (Frankfurt, Seoul) requires the endpoint to be identified.
+Expect better performance from direct connections \u2014traceroute will give you some insight.
+
+Examples:
+
+The default endpoint:
+
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.amazonaws.com</value>
+</property>
+```
+
+Frankfurt
+
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.eu-central-1.amazonaws.com</value>
+</property>
+```
+
+Seoul
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.ap-northeast-2.amazonaws.com</value>
+</property>
+```
+
+If the wrong endpoint is used, the request may fail. This may be reported as a 301/redirect error,
+or as a 400 Bad Request.
+
### S3AFastOutputStream
**Warning: NEW in hadoop 2.7. UNSTABLE, EXPERIMENTAL: use at own risk**
@@ -819,8 +873,6 @@ of environment-variable authentication by attempting to use the `hdfs fs` comman
to read or write data on S3. That is: comment out the `fs.s3a` secrets and rely on
the environment variables.
-S3 Frankfurt is a special case. It uses the V4 authentication API.
-
### Authentication failures running on Java 8u60+
A change in the Java 8 JVM broke some of the `toString()` string generation
@@ -829,6 +881,106 @@ generate authentication headers suitable for validation by S3.
Fix: make sure that the version of Joda Time is 2.8.1 or later.
+### "Bad Request" exception when working with AWS S3 Frankfurt, Seoul, or elsewhere
+
+
+S3 Frankfurt and Seoul *only* support
+[the V4 authentication API](http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html).
+
+Requests using the V2 API will be rejected with 400 `Bad Request`
+
+```
+$ bin/hadoop fs -ls s3a://frankfurt/
+WARN s3a.S3AFileSystem: Client: Amazon S3 error 400: 400 Bad Request; Bad Request (retryable)
+
+com.amazonaws.services.s3.model.AmazonS3Exception: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; Request ID: 923C5D9E75E44C06), S3 Extended Request ID: HDwje6k+ANEeDsM6aJ8+D5gUmNAMguOk2BvZ8PH3g9z0gpH+IuwT7N19oQOnIr5CIx7Vqb/uThE=
+ at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182)
+ at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770)
+ at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
+ at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
+ at com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1107)
+ at com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1070)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.verifyBucketExists(S3AFileSystem.java:307)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:284)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2793)
+ at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:101)
+ at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2830)
+ at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2812)
+ at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389)
+ at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356)
+ at org.apache.hadoop.fs.shell.PathData.expandAsGlob(PathData.java:325)
+ at org.apache.hadoop.fs.shell.Command.expandArgument(Command.java:235)
+ at org.apache.hadoop.fs.shell.Command.expandArguments(Command.java:218)
+ at org.apache.hadoop.fs.shell.FsCommand.processRawArguments(FsCommand.java:103)
+ at org.apache.hadoop.fs.shell.Command.run(Command.java:165)
+ at org.apache.hadoop.fs.FsShell.run(FsShell.java:315)
+ at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
+ at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90)
+ at org.apache.hadoop.fs.FsShell.main(FsShell.java:373)
+ls: doesBucketExist on frankfurt-new: com.amazonaws.services.s3.model.AmazonS3Exception:
+ Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request;
+```
+
+This happens when trying to work with any S3 service which only supports the
+"V4" signing API \u2014and he client is configured to use the default S3A service
+endpoint.
+
+The S3A client needs to be given the endpoint to use via the `fs.s3a.endpoint`
+property.
+
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.eu-central-1.amazonaws.com</value>
+</property>
+```
+
+### Error message "The bucket you are attempting to access must be addressed using the specified endpoint"
+
+This surfaces when `fs.s3a.endpoint` is configured to use S3 service endpoint
+which is neither the original AWS one, `s3.amazonaws.com` , nor the one where
+the bucket is hosted.
+
+```
+org.apache.hadoop.fs.s3a.AWSS3IOException: purging multipart uploads on landsat-pds:
+ com.amazonaws.services.s3.model.AmazonS3Exception:
+ The bucket you are attempting to access must be addressed using the specified endpoint.
+ Please send all future requests to this endpoint.
+ (Service: Amazon S3; Status Code: 301; Error Code: PermanentRedirect; Request ID: 5B7A5D18BE596E4B),
+ S3 Extended Request ID: uE4pbbmpxi8Nh7rycS6GfIEi9UH/SWmJfGtM9IeKvRyBPZp/hN7DbPyz272eynz3PEMM2azlhjE=:
+
+ at com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1182)
+ at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:770)
+ at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:489)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:310)
+ at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3785)
+ at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3738)
+ at com.amazonaws.services.s3.AmazonS3Client.listMultipartUploads(AmazonS3Client.java:2796)
+ at com.amazonaws.services.s3.transfer.TransferManager.abortMultipartUploads(TransferManager.java:1217)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initMultipartUploads(S3AFileSystem.java:454)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:289)
+ at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2715)
+ at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:96)
+ at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2749)
+ at org.apache.hadoop.fs.FileSystem$Cache.getUnique(FileSystem.java:2737)
+ at org.apache.hadoop.fs.FileSystem.newInstance(FileSystem.java:430)
+```
+
+1. Use the [Specific endpoint of the bucket's S3 service](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region)
+1. If not using "V4" authentication (see above), the original S3 endpoint
+can be used:
+
+```
+ <property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.amazonaws.com</value>
+ </property>
+```
+
+Using the explicit endpoint for the region is recommended for speed and the
+ability to use the V4 signing API.
+
## Visible S3 Inconsistency
Amazon S3 is *an eventually consistent object store*. That is: not a filesystem.
@@ -1124,7 +1276,43 @@ that the file `contract-test-options.xml` does not contain any
secret credentials itself. As the auth keys XML file is kept out of the
source code tree, it is not going to get accidentally committed.
-### Running Tests against non-AWS storage infrastructures
+
+### Running the Tests
+
+After completing the configuration, execute the test run through Maven.
+
+ mvn clean test
+
+It's also possible to execute multiple test suites in parallel by enabling the
+`parallel-tests` Maven profile. The tests spend most of their time blocked on
+network I/O with the S3 service, so running in parallel tends to complete full
+test runs faster.
+
+ mvn -Pparallel-tests clean test
+
+Some tests must run with exclusive access to the S3 bucket, so even with the
+`parallel-tests` profile enabled, several test suites will run in serial in a
+separate Maven execution step after the parallel tests.
+
+By default, the `parallel-tests` profile runs 4 test suites concurrently. This
+can be tuned by passing the `testsThreadCount` argument.
+
+ mvn -Pparallel-tests -DtestsThreadCount=8 clean test
+
+### Testing against different regions
+
+S3A can connect to different regions \u2014the tests support this. Simply
+define the target region in `contract-tests.xml` or any `auth-keys.xml`
+file referenced.
+
+```xml
+<property>
+ <name>fs.s3a.endpoint</name>
+ <value>s3.eu-central-1.amazonaws.com</value>
+</property>
+```
+This is used for all tests expect for scale tests using a Public CSV.gz file
+(see below)
### S3A session tests
@@ -1149,14 +1337,14 @@ An alternate endpoint may be defined in `test.fs.s3a.sts.endpoint`.
The default is ""; meaning "use the amazon default value".
-#### CSV Data source
+#### CSV Data source Tests
The `TestS3AInputStreamPerformance` tests require read access to a multi-MB
text file. The default file for these tests is one published by amazon,
[s3a://landsat-pds.s3.amazonaws.com/scene_list.gz](http://landsat-pds.s3.amazonaws.com/scene_list.gz).
This is a gzipped CSV index of other files which amazon serves for open use.
-The path to this object is set in the option `fs.s3a.scale.test.csvfile`:
+The path to this object is set in the option `fs.s3a.scale.test.csvfile`,
<property>
<name>fs.s3a.scale.test.csvfile</name>
@@ -1165,21 +1353,37 @@ The path to this object is set in the option `fs.s3a.scale.test.csvfile`:
1. If the option is not overridden, the default value is used. This
is hosted in Amazon's US-east datacenter.
-1. If the property is empty, tests which require it will be skipped.
+1. If `fs.s3a.scale.test.csvfile` is empty, tests which require it will be skipped.
1. If the data cannot be read for any reason then the test will fail.
1. If the property is set to a different path, then that data must be readable
and "sufficiently" large.
To test on different S3 endpoints, or alternate infrastructures supporting
-the same APIs, the option `fs.s3a.scale.test.csvfile` must therefore be
+the same APIs, the option `fs.s3a.scale.test.csvfile` must either be
set to " ", or an object of at least 10MB is uploaded to the object store, and
the `fs.s3a.scale.test.csvfile` option set to its path.
- <property>
- <name>fs.s3a.scale.test.csvfile</name>
- <value> </value>
- </property>
+```xml
+<property>
+ <name>fs.s3a.scale.test.csvfile</name>
+ <value> </value>
+</property>
+```
+(the reason the space or newline is needed is to add "an empty entry"; an empty
+`<value/>` would be considered undefined and pick up the default)
+
+*Note:* if using a test file in an S3 region requiring a different endpoint value
+set in `fs.s3a.endpoint`, define it in `fs.s3a.scale.test.csvfile.endpoint`.
+If the default CSV file is used, the tests will automatically use the us-east
+endpoint:
+
+```xml
+<property>
+ <name>fs.s3a.scale.test.csvfile.endpoint</name>
+ <value>s3.amazonaws.com</value>
+</property>
+```
#### Scale test operation count
@@ -1218,27 +1422,6 @@ smaller to achieve faster test runs.
<value>10240</value>
</property>
-### Running the Tests
-
-After completing the configuration, execute the test run through Maven.
-
- mvn clean test
-
-It's also possible to execute multiple test suites in parallel by enabling the
-`parallel-tests` Maven profile. The tests spend most of their time blocked on
-network I/O with the S3 service, so running in parallel tends to complete full
-test runs faster.
-
- mvn -Pparallel-tests clean test
-
-Some tests must run with exclusive access to the S3 bucket, so even with the
-`parallel-tests` profile enabled, several test suites will run in serial in a
-separate Maven execution step after the parallel tests.
-
-By default, the `parallel-tests` profile runs 4 test suites concurrently. This
-can be tuned by passing the `testsThreadCount` argument.
-
- mvn -Pparallel-tests -DtestsThreadCount=8 clean test
### Testing against non AWS S3 endpoints.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808876c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 04010d6..39c6028 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase;
import org.junit.Assert;
import org.junit.internal.AssumptionViolatedException;
import org.slf4j.Logger;
@@ -134,6 +135,20 @@ public class S3ATestUtils {
}
/**
+ * patch the endpoint option so that irrespective of where other tests
+ * are working, the IO performance tests can work with the landsat
+ * images.
+ * @param conf configuration to patch
+ */
+ public static void useCSVDataEndpoint(Configuration conf) {
+ String endpoint = conf.getTrimmed(S3AScaleTestBase.KEY_CSVTEST_ENDPOINT,
+ S3AScaleTestBase.DEFAULT_CSVTEST_ENDPOINT);
+ if (!endpoint.isEmpty()) {
+ conf.set(ENDPOINT, endpoint);
+ }
+ }
+
+ /**
* The exception to raise so as to exit fast from
* {@link #eventually(int, Callable)}.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808876c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index a25ca9c..4ff8407 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -133,6 +133,7 @@ public class TestS3AAWSCredentialsProvider {
AnonymousAWSCredentialsProvider.class.getName());
Path testFile = new Path(
conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE));
+ S3ATestUtils.useCSVDataEndpoint(conf);
FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf);
assertNotNull(fs);
assertTrue(fs instanceof S3AFileSystem);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808876c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index 21639b1..d861a16 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -82,7 +82,6 @@ public class S3AScaleTestBase extends Assert implements S3ATestConstants {
*/
public static final String KEY_CSVTEST_FILE =
S3A_SCALE_TEST + "csvfile";
-
/**
* Default path for the multi MB test file: {@value}.
*/
@@ -90,6 +89,20 @@ public class S3AScaleTestBase extends Assert implements S3ATestConstants {
= "s3a://landsat-pds/scene_list.gz";
/**
+ * Endpoint for the S3 CSV/scale tests. This defaults to
+ * being us-east.
+ */
+ public static final String KEY_CSVTEST_ENDPOINT =
+ S3A_SCALE_TEST + "csvfile.endpoint";
+
+ /**
+ * Endpoint for the S3 CSV/scale tests. This defaults to
+ * being us-east.
+ */
+ public static final String DEFAULT_CSVTEST_ENDPOINT =
+ "s3.amazonaws.com";
+
+ /**
* The default number of operations to perform: {@value}.
*/
public static final long DEFAULT_OPERATION_COUNT = 2005;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3808876c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
index bddd8e2..d6d9d66 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInputPolicy;
import org.apache.hadoop.fs.s3a.S3AInputStream;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -79,6 +80,7 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;
testDataAvailable = false;
} else {
+ S3ATestUtils.useCSVDataEndpoint(conf);
testData = new Path(testFile);
Path path = this.testData;
bindS3aFS(path);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-5521. Fix random failure of
TestCapacityScheduler#testKillAllAppsInQueue (sandflee via Varun Saxena)
Posted by cn...@apache.org.
YARN-5521. Fix random failure of TestCapacityScheduler#testKillAllAppsInQueue (sandflee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24249115
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24249115
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24249115
Branch: refs/heads/HADOOP-13345
Commit: 24249115bff3162c4202387da5bdd8eba13e6961
Parents: 83e57e0
Author: Varun Saxena <va...@apache.org>
Authored: Tue Aug 16 00:03:16 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 16 00:03:29 2016 +0530
----------------------------------------------------------------------
.../resourcemanager/scheduler/capacity/TestCapacityScheduler.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24249115/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 0b52b86..09c16d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -2184,6 +2184,7 @@ public class TestCapacityScheduler {
// check postconditions
rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
+ rm.waitForAppRemovedFromScheduler(app.getApplicationId());
appsInRoot = scheduler.getAppsInQueue("root");
assertTrue(appsInRoot.isEmpty());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: MAPREDUCE-6751. Add debug log
message when splitting is not possible due to unsplittable compression.
(Peter Vary via rchiang)
Posted by cn...@apache.org.
MAPREDUCE-6751. Add debug log message when splitting is not possible due to unsplittable compression. (Peter Vary via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c154abd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c154abd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c154abd
Branch: refs/heads/HADOOP-13345
Commit: 6c154abd33279475315b5f7f78dc47f1b0aa7028
Parents: b047bc7
Author: Ray Chiang <rc...@apache.org>
Authored: Tue Aug 16 12:13:22 2016 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Tue Aug 16 12:13:22 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/mapred/FileInputFormat.java | 7 +++++++
.../hadoop/mapreduce/lib/input/CombineFileInputFormat.java | 4 ++++
.../apache/hadoop/mapreduce/lib/input/FileInputFormat.java | 7 +++++++
3 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 2c58ebe..5803d60 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -369,6 +369,13 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
splitHosts[0], splitHosts[1]));
}
} else {
+ if (LOG.isDebugEnabled()) {
+ // Log only if the file is big enough to be splitted
+ if (length > Math.min(file.getBlockSize(), minSize)) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + file.getPath());
+ }
+ }
String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,0,length,clusterMap);
splits.add(makeSplit(path, 0, length, splitHosts[0], splitHosts[1]));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
index b2b7656..8f9699e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java
@@ -600,6 +600,10 @@ public abstract class CombineFileInputFormat<K, V>
if (!isSplitable) {
// if the file is not splitable, just create the one block with
// full file length
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + stat.getPath());
+ }
blocks = new OneBlockInfo[1];
fileSize = stat.getLen();
blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c154abd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index 0c5ede9..7ec882f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -422,6 +422,13 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
blkLocations[blkIndex].getCachedHosts()));
}
} else { // not splitable
+ if (LOG.isDebugEnabled()) {
+ // Log only if the file is big enough to be splitted
+ if (length > Math.min(file.getBlockSize(), minSize)) {
+ LOG.debug("File is not splittable so no parallelization "
+ + "is possible: " + file.getPath());
+ }
+ }
splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
blkLocations[0].getCachedHosts()));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDFS-10763. Open files can leak
permanently due to inconsistent lease update. Contributed by Kihwal Lee.
Posted by cn...@apache.org.
HDFS-10763. Open files can leak permanently due to inconsistent lease update. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/864f878d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/864f878d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/864f878d
Branch: refs/heads/HADOOP-13345
Commit: 864f878d5912c82f3204f1582cfb7eb7c9f1a1da
Parents: 03dea65
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 15 17:28:09 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 15 17:28:09 2016 -0500
----------------------------------------------------------------------
.../server/namenode/FSImageFormatPBINode.java | 10 ++---
.../hdfs/server/namenode/FSNamesystem.java | 3 +-
.../hdfs/server/namenode/TestLeaseManager.java | 47 ++++++++++++++++++++
3 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 1ecd947..1456ecf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -281,18 +281,14 @@ public final class FSImageFormatPBINode {
* Load the under-construction files section, and update the lease map
*/
void loadFilesUnderConstructionSection(InputStream in) throws IOException {
+ // Leases are added when the inode section is loaded. This section is
+ // still read in for compatibility reasons.
while (true) {
FileUnderConstructionEntry entry = FileUnderConstructionEntry
.parseDelimitedFrom(in);
if (entry == null) {
break;
}
- // update the lease manager
- INodeFile file = dir.getInode(entry.getInodeId()).asFile();
- FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature();
- Preconditions.checkState(uc != null); // file must be under-construction
- fsn.leaseManager.addLease(uc.getClientName(),
- entry.getInodeId());
}
}
@@ -371,6 +367,8 @@ public final class FSImageFormatPBINode {
if (f.hasFileUC()) {
INodeSection.FileUnderConstructionFeature uc = f.getFileUC();
file.toUnderConstruction(uc.getClientName(), uc.getClientMachine());
+ // update the lease manager
+ fsn.leaseManager.addLease(uc.getClientName(), file.getId());
if (blocks.length > 0) {
BlockInfo lastBlk = file.getLastBlock();
// replace the last block of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index be084c5..0621a77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3329,7 +3329,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new IOException("Cannot finalize file " + src
+ " because it is not under construction");
}
- leaseManager.removeLease(uc.getClientName(), pendingFile);
pendingFile.recordModification(latestSnapshot);
@@ -3340,6 +3339,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
allowCommittedBlock? numCommittedAllowed: 0,
blockManager.getMinReplication());
+ leaseManager.removeLease(uc.getClientName(), pendingFile);
+
// close file and persist block allocations for this file
closeFile(src, pendingFile);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/864f878d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
index f823745..6692090 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java
@@ -24,8 +24,14 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import com.google.common.collect.Lists;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.junit.Rule;
import org.junit.Test;
@@ -109,6 +115,47 @@ public class TestLeaseManager {
assertThat(lm.countPath(), is(1L));
}
+ /**
+ * Make sure the lease is restored even if only the inode has the record.
+ */
+ @Test
+ public void testLeaseRestorationOnRestart() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
+ .numDataNodes(1).build();
+ DistributedFileSystem dfs = cluster.getFileSystem();
+
+ // Create an empty file
+ String path = "/testLeaseRestorationOnRestart";
+ FSDataOutputStream out = dfs.create(new Path(path));
+
+ // Remove the lease from the lease manager, but leave it in the inode.
+ FSDirectory dir = cluster.getNamesystem().getFSDirectory();
+ INodeFile file = dir.getINode(path).asFile();
+ cluster.getNamesystem().leaseManager.removeLease(
+ file.getFileUnderConstructionFeature().getClientName(), file);
+
+ // Save a fsimage.
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNodeRpc().saveNamespace(0,0);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ // Restart the namenode.
+ cluster.restartNameNode(true);
+
+ // Check whether the lease manager has the lease
+ dir = cluster.getNamesystem().getFSDirectory();
+ file = dir.getINode(path).asFile();
+ assertTrue("Lease should exist.",
+ cluster.getNamesystem().leaseManager.getLease(file) != null);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
private static FSNamesystem makeMockFsNameSystem() {
FSDirectory dir = mock(FSDirectory.class);
FSNamesystem fsn = mock(FSNamesystem.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-9696. Garbage snapshot
records linger forever. Contributed by Kihwal Lee
Posted by cn...@apache.org.
HDFS-9696. Garbage snapshot records linger forever. Contributed by Kihwal Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83e57e08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83e57e08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83e57e08
Branch: refs/heads/HADOOP-13345
Commit: 83e57e083f2cf6c0de8a46966c5492faeabd8f2a
Parents: 9f29f42
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 15 13:01:23 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 15 13:01:23 2016 -0500
----------------------------------------------------------------------
.../server/namenode/FSImageFormatProtobuf.java | 6 ++-
.../hdfs/server/namenode/TestSaveNamespace.java | 42 ++++++++++++++++++++
2 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83e57e08/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 05087d1..7a81f9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -459,7 +459,11 @@ public final class FSImageFormatProtobuf {
this, summary, context, context.getSourceNamesystem());
snapshotSaver.serializeSnapshotSection(sectionOutputStream);
- snapshotSaver.serializeSnapshotDiffSection(sectionOutputStream);
+ // Skip snapshot-related sections when there is no snapshot.
+ if (context.getSourceNamesystem().getSnapshotManager()
+ .getNumSnapshots() > 0) {
+ snapshotSaver.serializeSnapshotDiffSection(sectionOutputStream);
+ }
snapshotSaver.serializeINodeReferenceSection(sectionOutputStream);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83e57e08/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
index a374585..0c86ef4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java
@@ -647,6 +647,48 @@ public class TestSaveNamespace {
}
}
+
+ @Test
+ public void testSkipSnapshotSection() throws Exception {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(new Configuration())
+ .numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ OutputStream out = null;
+ try {
+ String path = "/skipSnapshot";
+ out = fs.create(new Path(path));
+ out.close();
+
+ // add a bogus filediff
+ FSDirectory dir = cluster.getNamesystem().getFSDirectory();
+ INodeFile file = dir.getINode(path).asFile();
+ file.addSnapshotFeature(null).getDiffs()
+ .saveSelf2Snapshot(-1, file, null, false);
+
+ // make sure it has a diff
+ assertTrue("Snapshot fileDiff is missing.",
+ file.getFileWithSnapshotFeature().getDiffs() != null);
+
+ // saveNamespace
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNodeRpc().saveNamespace(0, 0);
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ // restart namenode
+ cluster.restartNameNode(true);
+ dir = cluster.getNamesystem().getFSDirectory();
+ file = dir.getINode(path).asFile();
+
+ // there should be no snapshot feature for the inode, when there is
+ // no snapshot.
+ assertTrue("There should be no snapshot feature for this INode.",
+ file.getFileWithSnapshotFeature() == null);
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
@Test
public void testSaveNamespaceBeforeShutdown() throws Exception {
Configuration conf = new HdfsConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-13410. RunJar adds the
content of the jar twice to the classpath (Yuanbo Liu via sjlee)
Posted by cn...@apache.org.
HADOOP-13410. RunJar adds the content of the jar twice to the classpath (Yuanbo Liu via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d3ea92f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d3ea92f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d3ea92f
Branch: refs/heads/HADOOP-13345
Commit: 4d3ea92f4fe4be2a0ee9849c65cb1c91b0c5711b
Parents: 874577a
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Aug 11 19:56:58 2016 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Thu Aug 11 19:56:58 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/RunJar.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d3ea92f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 19b51ad..d91a78b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -226,7 +226,7 @@ public class RunJar {
unJar(file, workDir);
- ClassLoader loader = createClassLoader(file, workDir);
+ ClassLoader loader = createClassLoader(workDir);
Thread.currentThread().setContextClassLoader(loader);
Class<?> mainClass = Class.forName(mainClassName, true, loader);
@@ -250,14 +250,13 @@ public class RunJar {
* the user jar as well as the HADOOP_CLASSPATH. Otherwise, it creates a
* classloader that simply adds the user jar to the classpath.
*/
- private ClassLoader createClassLoader(File file, final File workDir)
+ private ClassLoader createClassLoader(final File workDir)
throws MalformedURLException {
ClassLoader loader;
// see if the client classloader is enabled
if (useClientClassLoader()) {
StringBuilder sb = new StringBuilder();
sb.append(workDir).append("/").
- append(File.pathSeparator).append(file).
append(File.pathSeparator).append(workDir).append("/classes/").
append(File.pathSeparator).append(workDir).append("/lib/*");
// HADOOP_CLASSPATH is added to the client classpath
@@ -277,7 +276,6 @@ public class RunJar {
} else {
List<URL> classPath = new ArrayList<>();
classPath.add(new File(workDir + "/").toURI().toURL());
- classPath.add(file.toURI().toURL());
classPath.add(new File(workDir, "classes/").toURI().toURL());
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-13470.
GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)
Posted by cn...@apache.org.
HADOOP-13470. GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9336a049
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9336a049
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9336a049
Branch: refs/heads/HADOOP-13345
Commit: 9336a0495f99cd3fbc7ecef452eb37cfbaf57440
Parents: ef55fe1
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:24:54 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:24:54 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/test/GenericTestUtils.java | 25 ++++++-----
.../hadoop/test/TestGenericTestUtils.java | 44 ++++++++++++++++++++
2 files changed, 56 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9336a049/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 116a111..6b5135c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -42,10 +42,10 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
-import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
import org.junit.Assume;
@@ -275,36 +275,35 @@ public abstract class GenericTestUtils {
private StringWriter sw = new StringWriter();
private WriterAppender appender;
private Logger logger;
-
+
public static LogCapturer captureLogs(Log l) {
Logger logger = ((Log4JLogger)l).getLogger();
- LogCapturer c = new LogCapturer(logger);
- return c;
+ return new LogCapturer(logger);
+ }
+
+ public static LogCapturer captureLogs(org.slf4j.Logger logger) {
+ return new LogCapturer(toLog4j(logger));
}
-
private LogCapturer(Logger logger) {
this.logger = logger;
- Layout layout = Logger.getRootLogger().getAppender("stdout").getLayout();
- WriterAppender wa = new WriterAppender(layout, sw);
- logger.addAppender(wa);
+ this.appender = new WriterAppender(new PatternLayout(), sw);
+ logger.addAppender(appender);
}
-
+
public String getOutput() {
return sw.toString();
}
-
+
public void stopCapturing() {
logger.removeAppender(appender);
-
}
public void clearOutput() {
sw.getBuffer().setLength(0);
}
}
-
-
+
/**
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9336a049/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
index 8a7b5f6..86df5d5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
@@ -18,8 +18,16 @@
package org.apache.hadoop.test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
public class TestGenericTestUtils extends GenericTestUtils {
@Test
@@ -75,4 +83,40 @@ public class TestGenericTestUtils extends GenericTestUtils {
}
}
+ @Test(timeout = 10000)
+ public void testLogCapturer() {
+ final Log log = LogFactory.getLog(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(log);
+ final String infoMessage = "info message";
+ // test get output message
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
+ @Test(timeout = 10000)
+ public void testLogCapturerSlf4jLogger() {
+ final Logger logger = LoggerFactory.getLogger(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(logger);
+ final String infoMessage = "info message";
+ // test get output message
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-10745. Directly resolve paths
into INodesInPath. Contributed by Daryn Sharp.
Posted by cn...@apache.org.
HDFS-10745. Directly resolve paths into INodesInPath. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86939364
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86939364
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86939364
Branch: refs/heads/HADOOP-13345
Commit: 869393643de23dcb010cc33091c8eb398de0fd6c
Parents: e3037c5
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Aug 17 15:53:03 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Aug 17 15:53:03 2016 -0500
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 29 +++--
.../hdfs/server/namenode/FSDirAppendOp.java | 4 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 23 ++--
.../hdfs/server/namenode/FSDirDeleteOp.java | 4 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 7 +-
.../server/namenode/FSDirErasureCodingOp.java | 8 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 4 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 41 +++----
.../server/namenode/FSDirStatAndListingOp.java | 51 ++++-----
.../hdfs/server/namenode/FSDirSymlinkOp.java | 4 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 4 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 84 ++++-----------
.../hdfs/server/namenode/FSDirXAttrOp.java | 25 ++---
.../hdfs/server/namenode/FSDirectory.java | 107 +++++++++++++++----
.../hdfs/server/namenode/FSNamesystem.java | 49 +++------
.../hdfs/server/namenode/INodesInPath.java | 8 ++
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
17 files changed, 230 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index 296bed2..2153f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import java.io.IOException;
@@ -39,11 +38,11 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
@@ -64,11 +63,11 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
@@ -88,11 +87,11 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
@@ -112,11 +111,11 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
unprotectedRemoveAcl(fsd, iip);
} finally {
@@ -132,11 +131,11 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
List<AclEntry> newAcl = unprotectedSetAcl(fsd, src, aclSpec, false);
fsd.getEditLog().logSetAcl(src, newAcl);
@@ -150,17 +149,15 @@ class FSDirAclOp {
FSDirectory fsd, String src) throws IOException {
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
- String srcs = FSDirectory.normalizePath(src);
fsd.readLock();
try {
+ INodesInPath iip = fsd.resolvePath(pc, src);
+ src = iip.getPath();
// There is no real inode for the path ending in ".snapshot", so return a
// non-null, unpopulated AclStatus. This is similar to getFileInfo.
- if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR) &&
- fsd.getINode4DotSnapshot(srcs) != null) {
+ if (iip.isDotSnapshotDir() && fsd.getINode4DotSnapshot(iip) != null) {
return new AclStatus.Builder().owner("").group("").build();
}
- INodesInPath iip = fsd.getINodesInPath(srcs, true);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index daf5c71..3a5d7dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -87,8 +87,8 @@ final class FSDirAppendOp {
final String src;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg);
- final INodesInPath iip = fsd.getINodesInPath4Write(src);
+ final INodesInPath iip = fsd.resolvePathForWrite(pc, srcArg);
+ src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
final String path = iip.getPath();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 3dc1c30..e6d36b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -60,8 +60,8 @@ public class FSDirAttrOp {
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
unprotectedSetPermission(fsd, src, permission);
} finally {
@@ -81,8 +81,8 @@ public class FSDirAttrOp {
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
fsd.checkOwner(pc, iip);
if (!pc.isSuperUser()) {
if (username != null && !pc.getUser().equals(username)) {
@@ -108,8 +108,8 @@ public class FSDirAttrOp {
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
// Write access is required to set access and modification times
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
@@ -138,8 +138,8 @@ public class FSDirAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath4Write(src);
+ final INodesInPath iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
}
@@ -211,8 +211,7 @@ public class FSDirAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.readLock();
try {
- path = fsd.resolvePath(pc, path);
- final INodesInPath iip = fsd.getINodesInPath(path, false);
+ final INodesInPath iip = fsd.resolvePath(pc, path, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
}
@@ -232,8 +231,8 @@ public class FSDirAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, false);
+ final INodesInPath iip = fsd.resolvePath(pc, src, false);
+ src = iip.getPath();
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
index 58f7a61..13f1092 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
@@ -98,8 +98,8 @@ class FSDirDeleteOp {
FSDirectory fsd = fsn.getFSDirectory();
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath4Write(src, false);
+ final INodesInPath iip = fsd.resolvePathForWrite(pc, src, false);
+ src = iip.getPath();
if (!recursive && fsd.isNonEmptyDirectory(iip)) {
throw new PathIsNotEmptyDirectoryException(src + " is non empty");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index ba9e9d1..7501fc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -155,7 +155,8 @@ final class FSDirEncryptionZoneOp {
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg);
+ final INodesInPath iip = fsd.resolvePath(pc, srcArg);
+ src = iip.getPath();
final XAttr ezXAttr = fsd.ezManager.createEncryptionZone(src, suite,
version, keyName);
xAttrs.add(ezXAttr);
@@ -178,13 +179,11 @@ final class FSDirEncryptionZoneOp {
static Map.Entry<EncryptionZone, HdfsFileStatus> getEZForPath(
final FSDirectory fsd, final String srcArg, final FSPermissionChecker pc)
throws IOException {
- final String src;
final INodesInPath iip;
final EncryptionZone ret;
fsd.readLock();
try {
- src = fsd.resolvePath(pc, srcArg);
- iip = fsd.getINodesInPath(src, true);
+ iip = fsd.resolvePath(pc, srcArg);
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index d46342b..17544f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -73,12 +73,12 @@ final class FSDirErasureCodingOp {
FSPermissionChecker pc = null;
pc = fsn.getPermissionChecker();
FSDirectory fsd = fsn.getFSDirectory();
- src = fsd.resolvePath(pc, src);
final INodesInPath iip;
List<XAttr> xAttrs;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(src, false);
+ iip = fsd.resolvePathForWrite(pc, src, false);
+ src = iip.getPath();
xAttrs = createErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
} finally {
fsd.writeUnlock();
@@ -221,11 +221,9 @@ final class FSDirErasureCodingOp {
private static INodesInPath getINodesInPath(final FSNamesystem fsn,
final String srcArg) throws IOException {
- String src = srcArg;
final FSDirectory fsd = fsn.getFSDirectory();
final FSPermissionChecker pc = fsn.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
- INodesInPath iip = fsd.getINodesInPath(src, true);
+ INodesInPath iip = fsd.resolvePath(pc, srcArg);
if (fsn.isPermissionEnabled()) {
fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 1141422..8aac1f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -52,8 +52,8 @@ class FSDirMkdirOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- INodesInPath iip = fsd.getINodesInPath4Write(src);
+ INodesInPath iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index e9514ba..d90139b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -65,12 +65,14 @@ class FSDirRenameOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
HdfsFileStatus resultingStat = null;
- src = fsd.resolvePath(pc, src);
- dst = fsd.resolvePath(pc, dst);
+ // Rename does not operate on link targets
+ // Do not resolveLink when checking permissions of src and dst
+ INodesInPath srcIIP = fsd.resolvePathForWrite(pc, src, false);
+ INodesInPath dstIIP = fsd.resolvePathForWrite(pc, dst, false);
@SuppressWarnings("deprecation")
- final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
+ final boolean status = renameTo(fsd, pc, srcIIP, dstIIP, logRetryCache);
if (status) {
- INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
+ dstIIP = fsd.getINodesInPath(dstIIP.getPath(), false);
resultingStat = fsd.getAuditFileInfo(dstIIP);
}
return new RenameOldResult(status, resultingStat);
@@ -238,9 +240,8 @@ class FSDirRenameOp {
final FSPermissionChecker pc = fsd.getPermissionChecker();
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- src = fsd.resolvePath(pc, src);
- dst = fsd.resolvePath(pc, dst);
- renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
+ // returns resolved path
+ dst = renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dstIIP);
@@ -252,11 +253,13 @@ class FSDirRenameOp {
* @see {@link #unprotectedRenameTo(FSDirectory, String, String, INodesInPath,
* INodesInPath, long, BlocksMapUpdateInfo, Options.Rename...)}
*/
- static void renameTo(FSDirectory fsd, FSPermissionChecker pc, String src,
+ static String renameTo(FSDirectory fsd, FSPermissionChecker pc, String src,
String dst, BlocksMapUpdateInfo collectedBlocks, boolean logRetryCache,
Options.Rename... options) throws IOException {
- final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
- final INodesInPath dstIIP = fsd.getINodesInPath4Write(dst, false);
+ final INodesInPath srcIIP = fsd.resolvePathForWrite(pc, src, false);
+ final INodesInPath dstIIP = fsd.resolvePathForWrite(pc, dst, false);
+ src = srcIIP.getPath();
+ dst = dstIIP.getPath();
if (fsd.isPermissionEnabled()) {
// Rename does not operate on link targets
// Do not resolveLink when checking permissions of src and dst
@@ -283,6 +286,7 @@ class FSDirRenameOp {
fsd.writeUnlock();
}
fsd.getEditLog().logRename(src, dst, mtime, logRetryCache, options);
+ return dst;
}
/**
@@ -442,16 +446,17 @@ class FSDirRenameOp {
@Deprecated
@SuppressWarnings("deprecation")
private static boolean renameTo(FSDirectory fsd, FSPermissionChecker pc,
- String src, String dst, boolean logRetryCache) throws IOException {
- // Rename does not operate on link targets
- // Do not resolveLink when checking permissions of src and dst
- // Check write access to parent of src
- final INodesInPath srcIIP = fsd.getINodesInPath4Write(src, false);
+ INodesInPath srcIIP, INodesInPath dstIIP, boolean logRetryCache)
+ throws IOException {
+ String src = srcIIP.getPath();
+ String dst = dstIIP.getPath();
// Note: We should not be doing this. This is move() not renameTo().
- final String actualDst = fsd.isDir(dst) ?
- dst + Path.SEPARATOR + new Path(src).getName() : dst;
- final INodesInPath dstIIP = fsd.getINodesInPath4Write(actualDst, false);
+ if (fsd.isDir(dst)) {
+ dstIIP = INodesInPath.append(dstIIP, null, srcIIP.getLastLocalName());
+ }
+ final String actualDst = dstIIP.getPath();
if (fsd.isPermissionEnabled()) {
+ // Check write access to parent of src
fsd.checkPermission(pc, srcIIP, false, null, FsAction.WRITE, null, null,
false);
// Check write access to ancestor of dst
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 40efc8a..c9eedf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -55,11 +55,14 @@ class FSDirStatAndListingOp {
final String startAfterString = DFSUtil.bytes2String(startAfter);
String src = null;
+ final INodesInPath iip;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, srcArg);
+ iip = fsd.resolvePath(pc, srcArg);
+ src = iip.getPath();
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
+ iip = fsd.getINodesInPath(src, true);
}
// Get file name when startAfter is an INodePath
@@ -75,7 +78,6 @@ class FSDirStatAndListingOp {
}
}
- final INodesInPath iip = fsd.getINodesInPath(src, true);
boolean isSuperUser = true;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
@@ -108,8 +110,8 @@ class FSDirStatAndListingOp {
}
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, srcArg);
- final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
+ final INodesInPath iip = fsd.resolvePath(pc, srcArg, resolveLink);
+ src = iip.getPath();
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
src = FSDirectory.resolvePath(srcArg, fsd);
@@ -123,8 +125,7 @@ class FSDirStatAndListingOp {
*/
static boolean isFileClosed(FSDirectory fsd, String src) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, true);
+ final INodesInPath iip = fsd.resolvePath(pc, src);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
}
@@ -134,8 +135,7 @@ class FSDirStatAndListingOp {
static ContentSummary getContentSummary(
FSDirectory fsd, String src) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, false);
+ final INodesInPath iip = fsd.resolvePath(pc, src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,
FsAction.READ_EXECUTE);
@@ -160,8 +160,8 @@ class FSDirStatAndListingOp {
boolean isReservedName = FSDirectory.isReservedRawName(src);
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, true);
+ final INodesInPath iip = fsd.resolvePath(pc, src);
+ src = iip.getPath();
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
@@ -390,24 +390,20 @@ class FSDirStatAndListingOp {
static HdfsFileStatus getFileInfo(
FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
throws IOException {
- String srcs = FSDirectory.normalizePath(src);
- if (FSDirectory.isExactReservedName(src)) {
- return FSDirectory.DOT_RESERVED_STATUS;
- }
-
- if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
- if (fsd.getINode4DotSnapshot(srcs) != null) {
- return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
- HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
- HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
- }
- return null;
- }
-
fsd.readLock();
try {
- final INodesInPath iip = fsd.getINodesInPath(srcs, resolveLink);
- return getFileInfo(fsd, src, iip, isRawPath, true);
+ HdfsFileStatus status = null;
+ final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
+ if (FSDirectory.isExactReservedName(iip.getPathComponents())) {
+ status = FSDirectory.DOT_RESERVED_STATUS;
+ } else if (iip.isDotSnapshotDir()) {
+ if (fsd.getINode4DotSnapshot(iip) != null) {
+ status = FSDirectory.DOT_SNAPSHOT_DIR_STATUS;
+ }
+ } else {
+ status = getFileInfo(fsd, src, iip, isRawPath, true);
+ }
+ return status;
} finally {
fsd.readUnlock();
}
@@ -624,8 +620,7 @@ class FSDirStatAndListingOp {
final INodesInPath iip;
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src);
- iip = fsd.getINodesInPath(src, false);
+ iip = fsd.resolvePath(pc, src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,
FsAction.READ_EXECUTE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index e78c7b5..4d32993 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -56,8 +56,8 @@ class FSDirSymlinkOp {
INodesInPath iip;
fsd.writeLock();
try {
- link = fsd.resolvePath(pc, link);
- iip = fsd.getINodesInPath4Write(link, false);
+ iip = fsd.resolvePathForWrite(pc, link, false);
+ link = iip.getPath();
if (!createParent) {
fsd.verifyParentDir(iip, link);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index 8ac59f6..19518b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -77,8 +77,8 @@ final class FSDirTruncateOp {
Block truncateBlock = null;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg);
- iip = fsd.getINodesInPath4Write(src, true);
+ iip = fsd.resolvePathForWrite(pc, srcArg);
+ src = iip.getPath();
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 0f55875..8429024 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -119,25 +118,10 @@ class FSDirWriteFileOp {
static void abandonBlock(
FSDirectory fsd, FSPermissionChecker pc, ExtendedBlock b, long fileId,
String src, String holder) throws IOException {
- src = fsd.resolvePath(pc, src);
-
- final INode inode;
- final INodesInPath iip;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = fsd.getINodesInPath(src, true);
- inode = iip.getLastINode();
- } else {
- inode = fsd.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
+ final INodesInPath iip = fsd.resolvePath(pc, src, fileId);
+ src = iip.getPath();
FSNamesystem fsn = fsd.getFSNamesystem();
- final INodeFile file = fsn.checkLease(src, holder, inode, fileId);
+ final INodeFile file = fsn.checkLease(iip, holder, fileId);
Preconditions.checkState(file.isUnderConstruction());
if (file.isStriped()) {
return; // do not abandon block for striped file
@@ -184,8 +168,8 @@ class FSDirWriteFileOp {
String clientMachine;
final boolean isStriped;
- src = fsn.dir.resolvePath(pc, src);
- FileState fileState = analyzeFileState(fsn, src, fileId, clientName,
+ INodesInPath iip = fsn.dir.resolvePath(pc, src, fileId);
+ FileState fileState = analyzeFileState(fsn, iip, fileId, clientName,
previous, onRetryBlock);
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
// This is a retry. No need to generate new locations.
@@ -243,7 +227,8 @@ class FSDirWriteFileOp {
// Run the full analysis again, since things could have changed
// while chooseTarget() was executing.
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
- FileState fileState = analyzeFileState(fsn, src, fileId, clientName,
+ INodesInPath iip = fsn.dir.resolvePath(null, src, fileId);
+ FileState fileState = analyzeFileState(fsn, iip, fileId, clientName,
previous, onRetryBlock);
final INodeFile pendingFile = fileState.inode;
src = fileState.path;
@@ -357,8 +342,8 @@ class FSDirWriteFileOp {
boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
- src = fsd.resolvePath(pc, src);
- INodesInPath iip = fsd.getINodesInPath4Write(src);
+ INodesInPath iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
// Verify that the destination does not exist as a directory already.
final INode inode = iip.getLastINode();
@@ -467,8 +452,7 @@ class FSDirWriteFileOp {
CryptoProtocolVersion[] supportedVersions)
throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
- src = fsd.resolvePath(pc, src);
- INodesInPath iip = fsd.getINodesInPath4Write(src);
+ INodesInPath iip = fsd.resolvePathForWrite(pc, src);
// Nothing to do if the path is not within an EZ
final EncryptionZone zone = FSDirEncryptionZoneOp.getEZForPath(fsd, iip);
if (zone == null) {
@@ -625,11 +609,11 @@ class FSDirWriteFileOp {
}
private static FileState analyzeFileState(
- FSNamesystem fsn, String src, long fileId, String clientName,
+ FSNamesystem fsn, INodesInPath iip, long fileId, String clientName,
ExtendedBlock previous, LocatedBlock[] onRetryBlock)
throws IOException {
assert fsn.hasReadLock();
-
+ String src = iip.getPath();
checkBlock(fsn, previous);
onRetryBlock[0] = null;
fsn.checkNameNodeSafeMode("Cannot add block to " + src);
@@ -638,24 +622,7 @@ class FSDirWriteFileOp {
fsn.checkFsObjectLimit();
Block previousBlock = ExtendedBlock.getLocalBlock(previous);
- final INode inode;
- final INodesInPath iip;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = fsn.dir.getINodesInPath4Write(src);
- inode = iip.getLastINode();
- } else {
- // Newer clients pass the inode ID, so we can just get the inode
- // directly.
- inode = fsn.dir.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
- final INodeFile file = fsn.checkLease(src, clientName, inode, fileId);
+ final INodeFile file = fsn.checkLease(iip, clientName, fileId);
BlockInfo lastBlockInFile = file.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
@@ -734,33 +701,22 @@ class FSDirWriteFileOp {
src + " for " + holder);
}
checkBlock(fsn, last);
- src = fsn.dir.resolvePath(pc, src);
- return completeFileInternal(fsn, src, holder,
+ INodesInPath iip = fsn.dir.resolvePath(pc, src, fileId);
+ return completeFileInternal(fsn, iip, holder,
ExtendedBlock.getLocalBlock(last), fileId);
}
private static boolean completeFileInternal(
- FSNamesystem fsn, String src, String holder, Block last, long fileId)
+ FSNamesystem fsn, INodesInPath iip,
+ String holder, Block last, long fileId)
throws IOException {
assert fsn.hasWriteLock();
+ final String src = iip.getPath();
final INodeFile pendingFile;
- final INodesInPath iip;
INode inode = null;
try {
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- iip = fsn.dir.getINodesInPath(src, true);
- inode = iip.getLastINode();
- } else {
- inode = fsn.dir.getInode(fileId);
- iip = INodesInPath.fromINode(inode);
- if (inode != null) {
- src = iip.getPath();
- }
- }
- pendingFile = fsn.checkLease(src, holder, inode, fileId);
+ inode = iip.getLastINode();
+ pendingFile = fsn.checkLease(iip, holder, fileId);
} catch (LeaseExpiredException lee) {
if (inode != null && inode.isFile() &&
!inode.asFile().isUnderConstruction()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 668e9e8..746fdb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -67,13 +67,13 @@ class FSDirXAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
- src = fsd.resolvePath(pc, src);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
INodesInPath iip;
fsd.writeLock();
try {
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
checkXAttrChangeAccess(fsd, iip, xAttr, pc);
unprotectedSetXAttrs(fsd, src, xAttrs, flag);
} finally {
@@ -94,12 +94,11 @@ class FSDirXAttrOp {
if (!getAll) {
XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
}
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, true);
+ final INodesInPath iip = fsd.resolvePath(pc, src);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
}
- List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
+ List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, iip);
List<XAttr> filteredAll = XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
@@ -134,13 +133,12 @@ class FSDirXAttrOp {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
final FSPermissionChecker pc = fsd.getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
- src = fsd.resolvePath(pc, src);
- final INodesInPath iip = fsd.getINodesInPath(src, true);
+ final INodesInPath iip = fsd.resolvePath(pc, src);
if (fsd.isPermissionEnabled()) {
/* To access xattr names, you need EXECUTE in the owning directory. */
fsd.checkParentAccess(pc, iip, FsAction.EXECUTE);
}
- final List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
+ final List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, iip);
return XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
}
@@ -167,8 +165,8 @@ class FSDirXAttrOp {
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src);
- iip = fsd.getINodesInPath4Write(src);
+ iip = fsd.resolvePathForWrite(pc, src);
+ src = iip.getPath();
checkXAttrChangeAccess(fsd, iip, xAttr, pc);
List<XAttr> removedXAttrs = unprotectedRemoveXAttrs(fsd, src, xAttrs);
@@ -427,12 +425,11 @@ class FSDirXAttrOp {
}
}
- private static List<XAttr> getXAttrs(FSDirectory fsd,
- String src) throws IOException {
- String srcs = FSDirectory.normalizePath(src);
+ private static List<XAttr> getXAttrs(FSDirectory fsd, INodesInPath iip)
+ throws IOException {
fsd.readLock();
try {
- INodesInPath iip = fsd.getINodesInPath(srcs, true);
+ String src = iip.getPath();
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getPathSnapshotId();
return XAttrStorage.readINodeXAttrs(fsd.getAttributes(src,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 8321fda..b6652e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -132,6 +132,11 @@ public class FSDirectory implements Closeable {
null, null, null, HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+ public final static HdfsFileStatus DOT_SNAPSHOT_DIR_STATUS =
+ new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+
INodeDirectory rootDir;
private final FSNamesystem namesystem;
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
@@ -526,12 +531,66 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- String resolvePath(FSPermissionChecker pc, String path)
- throws FileNotFoundException, AccessControlException {
- if (isReservedRawName(path) && isPermissionEnabled) {
+ INodesInPath resolvePath(FSPermissionChecker pc, String src)
+ throws UnresolvedLinkException, FileNotFoundException,
+ AccessControlException {
+ return resolvePath(pc, src, true);
+ }
+
+ INodesInPath resolvePath(FSPermissionChecker pc, String src,
+ boolean resolveLink) throws UnresolvedLinkException,
+ FileNotFoundException, AccessControlException {
+ byte[][] components = INode.getPathComponents(src);
+ if (isPermissionEnabled && pc != null && isReservedRawName(components)) {
pc.checkSuperuserPrivilege();
}
- return resolvePath(path, this);
+ components = resolveComponents(components, this);
+ return INodesInPath.resolve(rootDir, components, resolveLink);
+ }
+
+ INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src)
+ throws UnresolvedLinkException, FileNotFoundException,
+ AccessControlException {
+ return resolvePathForWrite(pc, src, true);
+ }
+
+ INodesInPath resolvePathForWrite(FSPermissionChecker pc, String src,
+ boolean resolveLink) throws UnresolvedLinkException,
+ FileNotFoundException, AccessControlException {
+ INodesInPath iip = resolvePath(pc, src, resolveLink);
+ if (iip.isSnapshot()) {
+ throw new SnapshotAccessControlException(
+ "Modification on a read-only snapshot is disallowed");
+ }
+ return iip;
+ }
+
+ INodesInPath resolvePath(FSPermissionChecker pc, String src, long fileId)
+ throws UnresolvedLinkException, FileNotFoundException,
+ AccessControlException {
+ // Older clients may not have given us an inode ID to work with.
+ // In this case, we have to try to resolve the path and hope it
+ // hasn't changed or been deleted since the file was opened for write.
+ INodesInPath iip;
+ if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
+ iip = resolvePath(pc, src);
+ } else {
+ INode inode = getInode(fileId);
+ if (inode == null) {
+ iip = INodesInPath.fromComponents(INode.getPathComponents(src));
+ } else {
+ iip = INodesInPath.fromINode(inode);
+ }
+ }
+ return iip;
+ }
+
+ // this method can be removed after IIP is used more extensively
+ static String resolvePath(String src,
+ FSDirectory fsd) throws FileNotFoundException {
+ byte[][] pathComponents = INode.getPathComponents(src);
+ pathComponents = resolveComponents(pathComponents, fsd);
+ return DFSUtil.byteArray2PathString(pathComponents);
}
/**
@@ -1305,6 +1364,12 @@ public class FSDirectory implements Closeable {
return CHECK_RESERVED_FILE_NAMES && src.equals(DOT_RESERVED_PATH_PREFIX);
}
+ public static boolean isExactReservedName(byte[][] components) {
+ return CHECK_RESERVED_FILE_NAMES &&
+ (components.length == 2) &&
+ isReservedName(components);
+ }
+
static boolean isReservedRawName(String src) {
return src.startsWith(DOT_RESERVED_PATH_PREFIX +
Path.SEPARATOR + RAW_STRING);
@@ -1316,11 +1381,17 @@ public class FSDirectory implements Closeable {
}
static boolean isReservedName(byte[][] components) {
- return (components.length > 2) &&
+ return (components.length > 1) &&
Arrays.equals(INodeDirectory.ROOT_NAME, components[0]) &&
Arrays.equals(DOT_RESERVED, components[1]);
}
+ static boolean isReservedRawName(byte[][] components) {
+ return (components.length > 2) &&
+ isReservedName(components) &&
+ Arrays.equals(RAW, components[2]);
+ }
+
/**
* Resolve a /.reserved/... path to a non-reserved path.
* <p/>
@@ -1339,19 +1410,18 @@ public class FSDirectory implements Closeable {
* /.reserved/raw/a/b/c is equivalent (they both refer to the same
* unencrypted file).
*
- * @param src path that is being processed
+ * @param pathComponents to be resolved
* @param fsd FSDirectory
* @return if the path indicates an inode, return path after replacing up to
* <inodeid> with the corresponding path of the inode, else the path
- * in {@code src} as is. If the path refers to a path in the "raw"
- * directory, return the non-raw pathname.
+ * in {@code pathComponents} as is. If the path refers to a path in
+ * the "raw" directory, return the non-raw pathname.
* @throws FileNotFoundException if inodeid is invalid
*/
- static String resolvePath(String src,
+ static byte[][] resolveComponents(byte[][] pathComponents,
FSDirectory fsd) throws FileNotFoundException {
- byte[][] pathComponents = INode.getPathComponents(src);
final int nComponents = pathComponents.length;
- if (!isReservedName(pathComponents)) {
+ if (nComponents < 3 || !isReservedName(pathComponents)) {
/* This is not a /.reserved/ path so do nothing. */
} else if (Arrays.equals(DOT_INODES, pathComponents[2])) {
/* It's a /.reserved/.inodes path. */
@@ -1372,9 +1442,7 @@ public class FSDirectory implements Closeable {
}
}
}
- // this double conversion will be unnecessary when resolving returns
- // INodesInPath (needs components byte[][])
- return DFSUtil.byteArray2PathString(pathComponents);
+ return pathComponents;
}
private static byte[][] resolveDotInodesPath(
@@ -1428,15 +1496,12 @@ public class FSDirectory implements Closeable {
return components;
}
- INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
+ INode getINode4DotSnapshot(INodesInPath iip) throws UnresolvedLinkException {
Preconditions.checkArgument(
- src.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR),
- "%s does not end with %s", src, HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
-
- final String dirPath = normalizePath(src.substring(0,
- src.length() - HdfsConstants.DOT_SNAPSHOT_DIR.length()));
+ iip.isDotSnapshotDir(), "%s does not end with %s",
+ iip.getPath(), HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR);
- final INode node = this.getINode(dirPath);
+ final INode node = iip.getINode(-2);
if (node != null && node.isDirectory()
&& node.asDirectory().isSnapshottable()) {
return node;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0621a77..29f09b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1813,8 +1813,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* HDFS-7463. A better fix is to change the edit log of SetTime to
* use inode id instead of a path.
*/
- src = dir.resolvePath(pc, srcArg);
- final INodesInPath iip = dir.getINodesInPath(src, true);
+ final INodesInPath iip = dir.resolvePath(pc, srcArg);
+ src = iip.getPath();
+
INode inode = iip.getLastINode();
boolean updateAccessTime = inode != null &&
now > inode.getAccessTime() + dir.getAccessTimePrecision();
@@ -2311,8 +2312,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot recover the lease of " + src);
- src = dir.resolvePath(pc, src);
- final INodesInPath iip = dir.getINodesInPath4Write(src);
+ final INodesInPath iip = dir.resolvePathForWrite(pc, src);
+ src = iip.getPath();
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (!inode.isUnderConstruction()) {
return true;
@@ -2556,20 +2557,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
checkOperation(OperationCategory.READ);
//check safe mode
checkNameNodeSafeMode("Cannot add datanode; src=" + src + ", blk=" + blk);
- src = dir.resolvePath(pc, src);
+ final INodesInPath iip = dir.resolvePath(pc, src, fileId);
+ src = iip.getPath();
//check lease
- final INode inode;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- inode = dir.getINode(src);
- } else {
- inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
- }
- final INodeFile file = checkLease(src, clientName, inode, fileId);
+ final INodeFile file = checkLease(iip, clientName, fileId);
clientMachine = file.getFileUnderConstructionFeature().getClientMachine();
clientnode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
preferredblocksize = file.getPreferredBlockSize();
@@ -2627,8 +2619,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
: "Holder " + holder + " does not have any open files.");
}
- INodeFile checkLease(String src, String holder, INode inode, long fileId)
+ INodeFile checkLease(INodesInPath iip, String holder, long fileId)
throws LeaseExpiredException, FileNotFoundException {
+ String src = iip.getPath();
+ INode inode = iip.getLastINode();
assert hasReadLock();
if (inode == null) {
throw new FileNotFoundException("File does not exist: "
@@ -3075,18 +3069,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot fsync file " + src);
- src = dir.resolvePath(pc, src);
- final INode inode;
- if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
- // Older clients may not have given us an inode ID to work with.
- // In this case, we have to try to resolve the path and hope it
- // hasn't changed or been deleted since the file was opened for write.
- inode = dir.getINode(src);
- } else {
- inode = dir.getInode(fileId);
- if (inode != null) src = inode.getFullPathName();
- }
- final INodeFile pendingFile = checkLease(src, clientName, inode, fileId);
+ INodesInPath iip = dir.resolvePath(pc, src, fileId);
+ src = iip.getPath();
+ final INodeFile pendingFile = checkLease(iip, clientName, fileId);
if (lastBlockLength > 0) {
pendingFile.getFileUnderConstructionFeature().updateLengthOfLastBlock(
pendingFile, lastBlockLength);
@@ -6874,17 +6859,17 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void checkAccess(String src, FsAction mode) throws IOException {
checkOperation(OperationCategory.READ);
+ FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
- src = FSDirectory.resolvePath(src, dir);
- final INodesInPath iip = dir.getINodesInPath(src, true);
+ final INodesInPath iip = dir.resolvePath(pc, src);
+ src = iip.getPath();
INode inode = iip.getLastINode();
if (inode == null) {
throw new FileNotFoundException("Path not found");
}
if (isPermissionEnabled) {
- FSPermissionChecker pc = getPermissionChecker();
dir.checkPathAccess(pc, iip, mode);
}
} catch (AccessControlException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
index a7b772c..86cab28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java
@@ -73,6 +73,10 @@ public class INodesInPath {
return new INodesInPath(inodes, path);
}
+ static INodesInPath fromComponents(byte[][] components) {
+ return new INodesInPath(new INode[components.length], components);
+ }
+
/**
* Given some components, create a path name.
* @param components The path components
@@ -434,6 +438,10 @@ public class INodesInPath {
return this.isSnapshot;
}
+ boolean isDotSnapshotDir() {
+ return isDotSnapshotDir(getLastLocalName());
+ }
+
private static String toString(INode inode) {
return inode == null? null: inode.getLocalName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86939364/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 859f72c..9fb4fe5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -24,7 +24,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -1303,7 +1303,7 @@ public class TestFsck {
when(fsName.getBlockManager()).thenReturn(blockManager);
when(fsName.getFSDirectory()).thenReturn(fsd);
when(fsd.getFSNamesystem()).thenReturn(fsName);
- when(fsd.getINodesInPath(anyString(), anyBoolean())).thenReturn(iip);
+ when(fsd.resolvePath(anyObject(), anyString())).thenReturn(iip);
when(blockManager.getDatanodeManager()).thenReturn(dnManager);
NamenodeFsck fsck = new NamenodeFsck(conf, namenode, nettop, pmap, out,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-10549. Correctly revoke file
leases when closing files. Contributed by Yiqun Lin.
Posted by cn...@apache.org.
HDFS-10549. Correctly revoke file leases when closing files. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2aa5e2c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2aa5e2c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2aa5e2c4
Branch: refs/heads/HADOOP-13345
Commit: 2aa5e2c40364cf1e90e6af7851801f5eda759002
Parents: c575231
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Aug 17 15:22:42 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Aug 17 15:52:38 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 22 +++++++++++-
.../hadoop/hdfs/DFSStripedOutputStream.java | 13 +++++++-
.../hadoop/hdfs/TestDistributedFileSystem.java | 35 +++++++++++++++++++-
4 files changed, 68 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa5e2c4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 833b1ee..ad86895 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -463,7 +463,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
/** Stop renewal of lease for the file. */
- void endFileLease(final long inodeId) throws IOException {
+ void endFileLease(final long inodeId) {
getLeaseRenewer().closeFile(inodeId, this);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa5e2c4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 93aee0e..a73ab95 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.server.namenode.RetryStartFileException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.util.ByteArrayManager;
import org.apache.hadoop.io.EnumSetWritable;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.token.Token;
@@ -732,6 +733,7 @@ public class DFSOutputStream extends FSOutputSummer
* resources associated with this stream.
*/
void abort() throws IOException {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
synchronized (this) {
if (isClosed()) {
return;
@@ -740,9 +742,19 @@ public class DFSOutputStream extends FSOutputSummer
new IOException("Lease timeout of "
+ (dfsClient.getConf().getHdfsTimeout() / 1000)
+ " seconds expired."));
- closeThreads(true);
+
+ try {
+ closeThreads(true);
+ } catch (IOException e) {
+ b.add(e);
+ }
}
+
dfsClient.endFileLease(fileId);
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
}
boolean isClosed() {
@@ -775,13 +787,21 @@ public class DFSOutputStream extends FSOutputSummer
*/
@Override
public void close() throws IOException {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
synchronized (this) {
try (TraceScope ignored = dfsClient.newPathTraceScope(
"DFSOutputStream#close", src)) {
closeImpl();
+ } catch (IOException e) {
+ b.add(e);
}
}
+
dfsClient.endFileLease(fileId);
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
}
protected synchronized void closeImpl() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa5e2c4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 502e0a5..d5d0dfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -798,6 +798,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
@Override
void abort() throws IOException {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
synchronized (this) {
if (isClosed()) {
return;
@@ -808,9 +809,19 @@ public class DFSStripedOutputStream extends DFSOutputStream {
+ (dfsClient.getConf().getHdfsTimeout() / 1000)
+ " seconds expired."));
}
- closeThreads(true);
+
+ try {
+ closeThreads(true);
+ } catch (IOException e) {
+ b.add(e);
+ }
}
+
dfsClient.endFileLease(fileId);
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa5e2c4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 50f9f36..69248d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.fs.StorageStatistics.LongStatistic;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -96,7 +97,6 @@ import org.junit.Assert;
import org.junit.Test;
import org.mockito.InOrder;
import org.mockito.internal.util.reflection.Whitebox;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1373,4 +1373,37 @@ public class TestDistributedFileSystem {
}
}
+ @Test
+ public void testDFSCloseFilesBeingWritten() throws Exception {
+ Configuration conf = getTestConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ DistributedFileSystem fileSys = cluster.getFileSystem();
+
+ // Create one file then delete it to trigger the FileNotFoundException
+ // when closing the file.
+ fileSys.create(new Path("/test/dfsclose/file-0"));
+ fileSys.delete(new Path("/test/dfsclose/file-0"), true);
+
+ DFSClient dfsClient = fileSys.getClient();
+ // Construct a new dfsClient to get the same LeaseRenewer instance,
+ // to avoid the original client being added to the leaseRenewer again.
+ DFSClient newDfsClient =
+ new DFSClient(cluster.getFileSystem(0).getUri(), conf);
+ LeaseRenewer leaseRenewer = newDfsClient.getLeaseRenewer();
+
+ dfsClient.closeAllFilesBeingWritten(false);
+ // Remove new dfsClient in leaseRenewer
+ leaseRenewer.closeClient(newDfsClient);
+
+ // The list of clients corresponding to this renewer should be empty
+ assertEquals(true, leaseRenewer.isEmpty());
+ assertEquals(true, dfsClient.isFilesBeingWrittenEmpty());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-11786. Fix Javadoc typos in
org.apache.hadoop.fs.FileSystem. Contributed by Andras Bokor.
Posted by cn...@apache.org.
HADOOP-11786. Fix Javadoc typos in org.apache.hadoop.fs.FileSystem. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c5752316
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c5752316
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c5752316
Branch: refs/heads/HADOOP-13345
Commit: c57523163f8936aec74ccf3b5a8db6f73b428bbf
Parents: 822d661
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Aug 17 14:58:50 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Aug 17 15:00:14 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 50 ++++++++++----------
1 file changed, 25 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c5752316/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 1fdfac0..c366729 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -934,13 +934,13 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
* @param f the file name to open
- * @param permission
+ * @param permission file permission
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @throws IOException
* @see #setPermission(Path, FsPermission)
*/
@@ -956,12 +956,12 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
* @param f the file name to open
- * @param permission
+ * @param permission file permission
* @param flags {@link CreateFlag}s to use for this stream.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @throws IOException
* @see #setPermission(Path, FsPermission)
*/
@@ -980,12 +980,12 @@ public abstract class FileSystem extends Configured implements Closeable {
* Create an FSDataOutputStream at the indicated Path with a custom
* checksum option
* @param f the file name to open
- * @param permission
+ * @param permission file permission
* @param flags {@link CreateFlag}s to use for this stream.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @param checksumOpt checksum parameter. If null, the values
* found in conf will be used.
* @throws IOException
@@ -1094,8 +1094,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @throws IOException
* @see #setPermission(Path, FsPermission)
*/
@@ -1112,13 +1112,13 @@ public abstract class FileSystem extends Configured implements Closeable {
* reporting. Same as create(), except fails if parent directory doesn't
* already exist.
* @param f the file name to open
- * @param permission
+ * @param permission file permission
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @throws IOException
* @see #setPermission(Path, FsPermission)
*/
@@ -1136,12 +1136,12 @@ public abstract class FileSystem extends Configured implements Closeable {
* reporting. Same as create(), except fails if parent directory doesn't
* already exist.
* @param f the file name to open
- * @param permission
+ * @param permission file permission
* @param flags {@link CreateFlag}s to use for this stream.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
- * @param blockSize
- * @param progress
+ * @param blockSize block size
+ * @param progress the progress reporter
* @throws IOException
* @see #setPermission(Path, FsPermission)
*/
@@ -1881,7 +1881,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Set the current working directory for the given file system. All relative
* paths will be resolved relative to it.
*
- * @param new_dir
+ * @param new_dir Path of new working directory
*/
public abstract void setWorkingDirectory(Path new_dir);
@@ -2324,7 +2324,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Set the verify checksum flag. This is only applicable if the
* corresponding FileSystem supports checksum. By default doesn't do anything.
- * @param verifyChecksum
+ * @param verifyChecksum Verify checksum flag
*/
public void setVerifyChecksum(boolean verifyChecksum) {
//doesn't do anything
@@ -2333,7 +2333,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Set the write checksum flag. This is only applicable if the
* corresponding FileSystem supports checksum. By default doesn't do anything.
- * @param writeChecksum
+ * @param writeChecksum Write checsum flag
*/
public void setWriteChecksum(boolean writeChecksum) {
//doesn't do anything
@@ -2369,8 +2369,8 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Set permission of a path.
- * @param p
- * @param permission
+ * @param p The path
+ * @param permission permission
*/
public void setPermission(Path p, FsPermission permission
) throws IOException {
@@ -2590,7 +2590,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* Refer to the HDFS extended attributes user documentation for details.
*
* @param path Path to get extended attributes
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map describing the XAttrs of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
@@ -2607,7 +2607,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*
* @param path Path to get extended attributes
* @param names XAttr names.
- * @return Map<String, byte[]> describing the XAttrs of the file or directory
+ * @return Map describing the XAttrs of the file or directory
* @throws IOException
*/
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
@@ -3520,7 +3520,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* In order to reset, we add up all the thread-local statistics data, and
* set rootData to the negative of that.
*
- * This may seem like a counterintuitive way to reset the statsitics. Why
+ * This may seem like a counterintuitive way to reset the statistics. Why
* can't we just zero out all the thread-local data? Well, thread-local
* data can only be modified by the thread that owns it. If we tried to
* modify the thread-local data from this thread, our modification might get
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HDFS-10737. disk balancer add
volume path to report command. Contributed by Yuanbo Liu.
Posted by cn...@apache.org.
HDFS-10737. disk balancer add volume path to report command. Contributed by Yuanbo Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f29f423
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f29f423
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f29f423
Branch: refs/heads/HADOOP-13345
Commit: 9f29f423e426e2d42e650cbed88e46c1c29a2a63
Parents: d677b68
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Aug 15 09:47:30 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Aug 15 09:47:30 2016 -0700
----------------------------------------------------------------------
.../server/diskbalancer/command/Command.java | 35 ++++++++++++++
.../diskbalancer/command/PlanCommand.java | 34 --------------
.../diskbalancer/command/ReportCommand.java | 4 +-
.../command/TestDiskBalancerCommand.java | 48 ++++++++++++++++++++
4 files changed, 86 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f29f423/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index de77365..3110c1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -33,13 +33,17 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
+import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerConstants;
import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
import org.apache.hadoop.hdfs.tools.DiskBalancer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -421,6 +425,37 @@ public abstract class Command extends Configured {
}
/**
+ * Reads the Physical path of the disks we are balancing. This is needed to
+ * make the disk balancer human friendly and not used in balancing.
+ *
+ * @param node - Disk Balancer Node.
+ */
+ protected void populatePathNames(
+ DiskBalancerDataNode node) throws IOException {
+ // if the cluster is a local file system, there is no need to
+ // invoke rpc call to dataNode.
+ if (getClusterURI().getScheme().startsWith("file")) {
+ return;
+ }
+ String dnAddress = node.getDataNodeIP() + ":" + node.getDataNodePort();
+ ClientDatanodeProtocol dnClient = getDataNodeProxy(dnAddress);
+ String volumeNameJson = dnClient.getDiskBalancerSetting(
+ DiskBalancerConstants.DISKBALANCER_VOLUME_NAME);
+ ObjectMapper mapper = new ObjectMapper();
+
+ @SuppressWarnings("unchecked")
+ Map<String, String> volumeMap =
+ mapper.readValue(volumeNameJson, HashMap.class);
+ for (DiskBalancerVolumeSet set : node.getVolumeSets().values()) {
+ for (DiskBalancerVolume vol : set.getVolumes()) {
+ if (volumeMap.containsKey(vol.getUuid())) {
+ vol.setPath(volumeMap.get(vol.getUuid()));
+ }
+ }
+ }
+ }
+
+ /**
* Set top number of nodes to be processed.
* */
public void setTopNodes(int topNodes) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f29f423/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index 3159312..72ad2c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -24,23 +24,13 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
-import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerConstants;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
.DiskBalancerDataNode;
-import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
-import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
- .DiskBalancerVolumeSet;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Step;
import org.apache.hadoop.hdfs.tools.DiskBalancer;
-import org.codehaus.jackson.map.ObjectMapper;
-
-import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
/**
* Class that implements Plan Command.
@@ -158,30 +148,6 @@ public class PlanCommand extends Command {
}
}
- /**
- * Reads the Physical path of the disks we are balancing. This is needed to
- * make the disk balancer human friendly and not used in balancing.
- *
- * @param node - Disk Balancer Node.
- */
- private void populatePathNames(DiskBalancerDataNode node) throws IOException {
- String dnAddress = node.getDataNodeIP() + ":" + node.getDataNodePort();
- ClientDatanodeProtocol dnClient = getDataNodeProxy(dnAddress);
- String volumeNameJson = dnClient.getDiskBalancerSetting(
- DiskBalancerConstants.DISKBALANCER_VOLUME_NAME);
- ObjectMapper mapper = new ObjectMapper();
-
- @SuppressWarnings("unchecked")
- Map<String, String> volumeMap =
- mapper.readValue(volumeNameJson, HashMap.class);
- for (DiskBalancerVolumeSet set : node.getVolumeSets().values()) {
- for (DiskBalancerVolume vol : set.getVolumes()) {
- if (volumeMap.containsKey(vol.getUuid())) {
- vol.setPath(volumeMap.get(vol.getUuid()));
- }
- }
- }
- }
/**
* Gets extended help for this command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f29f423/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
index 9f8e399..18dd77e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -130,7 +130,7 @@ public class ReportCommand extends Command {
}
private void handleNodeReport(final CommandLine cmd, StrBuilder result,
- final String nodeFormat, final String volumeFormat) {
+ final String nodeFormat, final String volumeFormat) throws Exception {
String outputLine = "";
/*
* get value that identifies a DataNode from command line, it could be UUID,
@@ -152,6 +152,8 @@ public class ReportCommand extends Command {
final String trueStr = "True";
final String falseStr = "False";
DiskBalancerDataNode dbdn = getNode(nodeVal);
+ // get storage path of datanode
+ populatePathNames(dbdn);
if (dbdn == null) {
outputLine = String.format(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f29f423/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index e55c418..0d24f28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.hdfs.server.diskbalancer.command;
+
import static org.hamcrest.CoreMatchers.allOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.is;
@@ -50,6 +51,7 @@ import static org.apache.hadoop.hdfs.tools.DiskBalancer.HELP;
import static org.apache.hadoop.hdfs.tools.DiskBalancer.NODE;
import static org.apache.hadoop.hdfs.tools.DiskBalancer.PLAN;
import static org.apache.hadoop.hdfs.tools.DiskBalancer.QUERY;
+import static org.apache.hadoop.hdfs.tools.DiskBalancer.REPORT;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
@@ -262,6 +264,41 @@ public class TestDiskBalancerCommand {
}
@Test(timeout = 60000)
+ public void testReportNodeWithoutJson() throws Exception {
+ String dataNodeUuid = cluster.getDataNodes().get(0).getDatanodeUuid();
+ final String planArg = String.format("-%s -%s %s",
+ REPORT, NODE, dataNodeUuid);
+ final String cmdLine = String
+ .format(
+ "hdfs diskbalancer %s", planArg);
+ List<String> outputs = runCommand(cmdLine, cluster);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(containsString("Reporting volume information for DataNode"),
+ containsString(dataNodeUuid))));
+ assertThat(
+ outputs.get(2),
+ is(allOf(containsString(dataNodeUuid),
+ containsString("2 volumes with node data density 0.00"))));
+ assertThat(
+ outputs.get(3),
+ is(allOf(containsString("DISK"),
+ containsString("/dfs/data/data1"),
+ containsString("0.00"),
+ containsString("1.00"))));
+ assertThat(
+ outputs.get(4),
+ is(allOf(containsString("DISK"),
+ containsString("/dfs/data/data2"),
+ containsString("0.00"),
+ containsString("1.00"))));
+ }
+
+ @Test(timeout = 60000)
public void testReadClusterFromJson() throws Exception {
ClusterConnector jsonConnector = ConnectorFactory.getCluster(clusterJson,
conf);
@@ -283,6 +320,17 @@ public class TestDiskBalancerCommand {
runCommand(cmdLine, cluster);
}
+ /* test -plan DataNodeID */
+ @Test(timeout = 60000)
+ public void testPlanJsonNode() throws Exception {
+ final String planArg = String.format("-%s %s", PLAN,
+ "a87654a9-54c7-4693-8dd9-c9c7021dc340");
+ final String cmdLine = String
+ .format(
+ "hdfs diskbalancer %s", planArg);
+ runCommand(cmdLine);
+ }
+
/* Test that illegal arguments are handled correctly*/
@Test(timeout = 60000)
public void testIllegalArgument() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDFS-10559. DiskBalancer: Use SHA1
for Plan ID. Contributed by Xiaobing Zhou.
Posted by cn...@apache.org.
HDFS-10559. DiskBalancer: Use SHA1 for Plan ID. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5628b36c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5628b36c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5628b36c
Branch: refs/heads/HADOOP-13345
Commit: 5628b36c0872d58c9b25f23da3dab4eafad9bca3
Parents: 02abd13
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Aug 15 20:10:21 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Aug 15 20:10:21 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/ClientDatanodeProtocol.java | 2 +-
.../ClientDatanodeProtocolTranslatorPB.java | 2 +-
.../src/main/proto/ClientDatanodeProtocol.proto | 2 +-
.../hadoop/hdfs/server/datanode/DiskBalancer.java | 14 +++++++-------
.../server/diskbalancer/command/CancelCommand.java | 2 +-
.../server/diskbalancer/command/ExecuteCommand.java | 2 +-
.../hdfs/server/diskbalancer/TestDiskBalancer.java | 4 ++--
.../hdfs/server/diskbalancer/TestDiskBalancerRPC.java | 2 +-
.../diskbalancer/TestDiskBalancerWithMockMover.java | 8 ++++----
9 files changed, 19 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
index 477d308..10041f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
@@ -175,7 +175,7 @@ public interface ClientDatanodeProtocol {
/**
* Cancel an executing plan.
*
- * @param planID - A SHA512 hash of the plan string.
+ * @param planID - A SHA-1 hash of the plan string.
*/
void cancelDiskBalancePlan(String planID) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
index 045ccd5..0cf006c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
@@ -369,7 +369,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
/**
* Cancels an executing disk balancer plan.
*
- * @param planID - A SHA512 hash of the plan string.
+ * @param planID - A SHA-1 hash of the plan string.
* @throws IOException on error
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
index 11d04af..e4333cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto
@@ -154,7 +154,7 @@ message GetBalancerBandwidthResponseProto {
* balancer plan to a data node.
*/
message SubmitDiskBalancerPlanRequestProto {
- required string planID = 1; // A hash of the plan like SHA512
+ required string planID = 1; // A hash of the plan like SHA-1
required string plan = 2; // Plan file data in Json format
optional uint64 planVersion = 3; // Plan version number
optional bool ignoreDateCheck = 4; // Ignore date checks on this plan.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index 2d126a0..523c0a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -157,7 +157,7 @@ public class DiskBalancer {
* Takes a client submitted plan and converts into a set of work items that
* can be executed by the blockMover.
*
- * @param planId - A SHA512 of the plan string
+ * @param planId - A SHA-1 of the plan string
* @param planVersion - version of the plan string - for future use.
* @param planFileName - Plan file name
* @param planData - Plan data in json format
@@ -308,7 +308,7 @@ public class DiskBalancer {
/**
* Verifies that user provided plan is valid.
*
- * @param planID - SHA 512 of the plan.
+ * @param planID - SHA-1 of the plan.
* @param planVersion - Version of the plan, for future use.
* @param plan - Plan String in Json.
* @param force - Skip verifying when the plan was generated.
@@ -345,15 +345,15 @@ public class DiskBalancer {
}
/**
- * Verifies that plan matches the SHA512 provided by the client.
+ * Verifies that plan matches the SHA-1 provided by the client.
*
- * @param planID - Sha512 Hex Bytes
+ * @param planID - SHA-1 Hex Bytes
* @param plan - Plan String
* @throws DiskBalancerException
*/
private NodePlan verifyPlanHash(String planID, String plan)
throws DiskBalancerException {
- final long sha512Length = 128;
+ final long sha1Length = 40;
if (plan == null || plan.length() == 0) {
LOG.error("Disk Balancer - Invalid plan.");
throw new DiskBalancerException("Invalid plan.",
@@ -361,8 +361,8 @@ public class DiskBalancer {
}
if ((planID == null) ||
- (planID.length() != sha512Length) ||
- !DigestUtils.sha512Hex(plan.getBytes(Charset.forName("UTF-8")))
+ (planID.length() != sha1Length) ||
+ !DigestUtils.shaHex(plan.getBytes(Charset.forName("UTF-8")))
.equalsIgnoreCase(planID)) {
LOG.error("Disk Balancer - Invalid plan hash.");
throw new DiskBalancerException("Invalid or mis-matched hash.",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
index 740292d..8b83e27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java
@@ -93,7 +93,7 @@ public class CancelCommand extends Command {
String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort();
Preconditions.checkNotNull(dataNodeAddress);
ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress);
- String planHash = DigestUtils.sha512Hex(planData);
+ String planHash = DigestUtils.shaHex(planData);
try {
dataNode.cancelDiskBalancePlan(planHash);
} catch (DiskBalancerException ex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
index ae9b092..f363c34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java
@@ -85,7 +85,7 @@ public class ExecuteCommand extends Command {
String dataNodeAddress = plan.getNodeName() + ":" + plan.getPort();
Preconditions.checkNotNull(dataNodeAddress);
ClientDatanodeProtocol dataNode = getDataNodeProxy(dataNodeAddress);
- String planHash = DigestUtils.sha512Hex(planData);
+ String planHash = DigestUtils.shaHex(planData);
try {
// TODO : Support skipping date check.
dataNode.submitDiskBalancerPlan(planHash, DiskBalancer.PLAN_VERSION,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
index b51d72f..dc177fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancer.java
@@ -191,7 +191,7 @@ public class TestDiskBalancer {
plan.setNodeUUID(dnNode.getDatanodeUuid());
plan.setTimeStamp(Time.now());
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
assertNotNull(plan.getVolumeSetPlans());
assertTrue(plan.getVolumeSetPlans().size() > 0);
plan.getVolumeSetPlans().get(0).setTolerancePercent(10);
@@ -307,7 +307,7 @@ public class TestDiskBalancer {
plan.setNodeUUID(dnNode.getDatanodeUuid());
plan.setTimeStamp(Time.now());
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
dataNode.submitDiskBalancerPlan(planID, 1, PLAN_FILE, planJson, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
index aa73016..12ce160 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerRPC.java
@@ -317,7 +317,7 @@ public class TestDiskBalancerRPC {
plan = new NodePlan(node.getDataNodeName(), node.getDataNodePort());
planner.balanceVolumeSet(node, node.getVolumeSets().get("DISK"), plan);
planVersion = 1;
- planHash = DigestUtils.sha512Hex(plan.toJson());
+ planHash = DigestUtils.shaHex(plan.toJson());
return this;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5628b36c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
index f54bb32..c362f49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDiskBalancerWithMockMover.java
@@ -125,7 +125,7 @@ public class TestDiskBalancerWithMockMover {
private void executeSubmitPlan(NodePlan plan, DiskBalancer balancer,
int version) throws IOException {
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
balancer.submitPlan(planID, version, PLAN_FILE, planJson, false);
}
@@ -214,7 +214,7 @@ public class TestDiskBalancerWithMockMover {
NodePlan plan = mockMoverHelper.getPlan();
DiskBalancer balancer = mockMoverHelper.getBalancer();
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
thrown.expect(DiskBalancerException.class);
thrown.expect(new DiskBalancerResultVerifier(DiskBalancerException
@@ -231,7 +231,7 @@ public class TestDiskBalancerWithMockMover {
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
char repChar = planID.charAt(0);
repChar++;
@@ -261,7 +261,7 @@ public class TestDiskBalancerWithMockMover {
String planJson = plan.toJson();
- String planID = DigestUtils.sha512Hex(planJson);
+ String planID = DigestUtils.shaHex(planJson);
balancer.cancelPlan(planID);
DiskBalancerWorkStatus status = balancer.queryWorkStatus();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN=5526.
DrainDispacher#serviceStop blocked if setDrainEventsOnStop invoked (sandflee
via Varun Saxena)
Posted by cn...@apache.org.
YARN=5526. DrainDispacher#serviceStop blocked if setDrainEventsOnStop invoked (sandflee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/913a8951
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/913a8951
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/913a8951
Branch: refs/heads/HADOOP-13345
Commit: 913a8951366a5d816bbae253b230250e0b74849e
Parents: 20f0eb8
Author: Varun Saxena <va...@apache.org>
Authored: Thu Aug 18 11:38:13 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 18 11:38:13 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/event/AsyncDispatcher.java | 6 +++++-
.../java/org/apache/hadoop/yarn/event/DrainDispatcher.java | 5 +++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913a8951/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 5dea1c8..89b5861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -148,7 +148,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);
synchronized (waitForDrained) {
- while (!drained && eventHandlingThread != null
+ while (!isDrained() && eventHandlingThread != null
&& eventHandlingThread.isAlive()
&& System.currentTimeMillis() < endTime) {
waitForDrained.wait(1000);
@@ -303,4 +303,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
protected boolean isEventThreadWaiting() {
return eventHandlingThread.getState() == Thread.State.WAITING;
}
+
+ protected boolean isDrained() {
+ return drained;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913a8951/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
index cf4b1b5..f769492 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
@@ -95,6 +95,11 @@ public class DrainDispatcher extends AsyncDispatcher {
}
@Override
+ protected boolean isDrained() {
+ return drained;
+ }
+
+ @Override
protected void serviceStop() throws Exception {
stopped = true;
super.serviceStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-10560. DiskBalancer: Reuse
ObjectMapper instance to improve the performance. Contributed by Yiqun Lin.
Posted by cn...@apache.org.
HDFS-10560. DiskBalancer: Reuse ObjectMapper instance to improve the performance. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b047bc72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b047bc72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b047bc72
Branch: refs/heads/HADOOP-13345
Commit: b047bc7270f3461156e4d08423c728ee9c67dba5
Parents: b427ce1
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Aug 16 10:20:08 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Aug 16 10:20:08 2016 -0700
----------------------------------------------------------------------
.../server/datanode/DiskBalancerWorkItem.java | 11 ++++++---
.../server/datanode/DiskBalancerWorkStatus.java | 26 +++++++++++---------
.../hdfs/server/datanode/DiskBalancer.java | 5 ++--
.../server/diskbalancer/command/Command.java | 6 +++--
.../connectors/JsonNodeConnector.java | 8 +++---
.../datamodel/DiskBalancerCluster.java | 11 ++++++---
.../datamodel/DiskBalancerVolume.java | 11 ++++++---
7 files changed, 46 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
index f46a987..592a89f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.htrace.fasterxml.jackson.annotation.JsonInclude;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.IOException;
@@ -34,6 +35,10 @@ import java.io.IOException;
@InterfaceStability.Unstable
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public class DiskBalancerWorkItem {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerWorkItem.class);
+
private long startTime;
private long secondsElapsed;
private long bytesToCopy;
@@ -74,8 +79,7 @@ public class DiskBalancerWorkItem {
*/
public static DiskBalancerWorkItem parseJson(String json) throws IOException {
Preconditions.checkNotNull(json);
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerWorkItem.class);
+ return READER.readValue(json);
}
/**
@@ -169,8 +173,7 @@ public class DiskBalancerWorkItem {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return MAPPER.writeValueAsString(this);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
index 14789b6..94bf6a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import org.codehaus.jackson.map.SerializationConfig;
import static org.codehaus.jackson.map.type.TypeFactory.defaultInstance;
@@ -38,6 +39,15 @@ import java.util.LinkedList;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DiskBalancerWorkStatus {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ private static final ObjectMapper MAPPER_WITH_INDENT_OUTPUT =
+ new ObjectMapper().enable(
+ SerializationConfig.Feature.INDENT_OUTPUT);
+ private static final ObjectReader READER_WORKSTATUS =
+ new ObjectMapper().reader(DiskBalancerWorkStatus.class);
+ private static final ObjectReader READER_WORKENTRY = new ObjectMapper()
+ .reader(defaultInstance().constructCollectionType(List.class,
+ DiskBalancerWorkEntry.class));
private final List<DiskBalancerWorkEntry> currentState;
private Result result;
@@ -92,10 +102,7 @@ public class DiskBalancerWorkStatus {
this.result = result;
this.planID = planID;
this.planFile = planFile;
- ObjectMapper mapper = new ObjectMapper();
- this.currentState = mapper.readValue(currentState,
- defaultInstance().constructCollectionType(
- List.class, DiskBalancerWorkEntry.class));
+ this.currentState = READER_WORKENTRY.readValue(currentState);
}
@@ -141,15 +148,11 @@ public class DiskBalancerWorkStatus {
* @throws IOException
**/
public String currentStateString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- mapper.enable(SerializationConfig.Feature.INDENT_OUTPUT);
- return mapper.writeValueAsString(currentState);
+ return MAPPER_WITH_INDENT_OUTPUT.writeValueAsString(currentState);
}
public String toJsonString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
-
+ return MAPPER.writeValueAsString(this);
}
/**
@@ -160,8 +163,7 @@ public class DiskBalancerWorkStatus {
*/
public static DiskBalancerWorkStatus parseJson(String json) throws
IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerWorkStatus.class);
+ return READER_WORKSTATUS.readValue(json);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
index 523c0a6..ec72d97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancer.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerConstants;
import org.apache.hadoop.hdfs.server.diskbalancer.DiskBalancerException;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Step;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.util.Time;
-import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -263,8 +263,7 @@ public class DiskBalancer {
for (Map.Entry<String, FsVolumeSpi> entry : volMap.entrySet()) {
pathMap.put(entry.getKey(), entry.getValue().getBasePath());
}
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(pathMap);
+ return JsonUtil.toJsonString(pathMap);
} catch (DiskBalancerException ex) {
throw ex;
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 3110c1a..a1c15ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +70,8 @@ import java.util.TreeSet;
* Common interface for command handling.
*/
public abstract class Command extends Configured {
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(HashMap.class);
static final Logger LOG = LoggerFactory.getLogger(Command.class);
private Map<String, String> validArgs = new HashMap<>();
private URI clusterURI;
@@ -441,11 +444,10 @@ public abstract class Command extends Configured {
ClientDatanodeProtocol dnClient = getDataNodeProxy(dnAddress);
String volumeNameJson = dnClient.getDiskBalancerSetting(
DiskBalancerConstants.DISKBALANCER_VOLUME_NAME);
- ObjectMapper mapper = new ObjectMapper();
@SuppressWarnings("unchecked")
Map<String, String> volumeMap =
- mapper.readValue(volumeNameJson, HashMap.class);
+ READER.readValue(volumeNameJson);
for (DiskBalancerVolumeSet set : node.getVolumeSets().values()) {
for (DiskBalancerVolume vol : set.getVolumes()) {
if (volumeMap.containsKey(vol.getUuid())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
index cc79648..b47beff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/connectors/JsonNodeConnector.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.hdfs.server.diskbalancer.connectors;
import com.google.common.base.Preconditions;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel
.DiskBalancerDataNode;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.File;
import java.net.URL;
@@ -35,6 +37,8 @@ import java.util.List;
public class JsonNodeConnector implements ClusterConnector {
private static final Logger LOG =
LoggerFactory.getLogger(JsonNodeConnector.class);
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerCluster.class);
private final URL clusterURI;
/**
@@ -56,9 +60,7 @@ public class JsonNodeConnector implements ClusterConnector {
Preconditions.checkNotNull(this.clusterURI);
String dataFilePath = this.clusterURI.getPath();
LOG.info("Reading cluster info from file : " + dataFilePath);
- ObjectMapper mapper = new ObjectMapper();
- DiskBalancerCluster cluster =
- mapper.readValue(new File(dataFilePath), DiskBalancerCluster.class);
+ DiskBalancerCluster cluster = READER.readValue(new File(dataFilePath));
String message = String.format("Found %d node(s)",
cluster.getNodes().size());
LOG.info(message);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
index 17a6ebb..8d7fb2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerCluster.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
import com.google.common.base.Preconditions;
+
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,9 +26,11 @@ import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.Planner;
import org.apache.hadoop.hdfs.server.diskbalancer.planner.PlannerFactory;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.File;
import java.io.IOException;
@@ -69,6 +72,8 @@ public class DiskBalancerCluster {
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerCluster.class);
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerCluster.class);
private final Set<String> exclusionList;
private final Set<String> inclusionList;
private ClusterConnector clusterConnector;
@@ -118,8 +123,7 @@ public class DiskBalancerCluster {
* @throws IOException
*/
public static DiskBalancerCluster parseJson(String json) throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerCluster.class);
+ return READER.readValue(json);
}
/**
@@ -232,8 +236,7 @@ public class DiskBalancerCluster {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return JsonUtil.toJsonString(this);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b047bc72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
index a6a8bdc..8b627b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.hdfs.server.diskbalancer.datamodel;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectReader;
import java.io.IOException;
@@ -30,6 +32,9 @@ import java.io.IOException;
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class DiskBalancerVolume {
+ private static final ObjectReader READER =
+ new ObjectMapper().reader(DiskBalancerVolume.class);
+
private String path;
private long capacity;
private String storageType;
@@ -58,8 +63,7 @@ public class DiskBalancerVolume {
* @throws IOException
*/
public static DiskBalancerVolume parseJson(String json) throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.readValue(json, DiskBalancerVolume.class);
+ return READER.readValue(json);
}
/**
@@ -305,8 +309,7 @@ public class DiskBalancerVolume {
* @throws IOException
*/
public String toJson() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
+ return JsonUtil.toJsonString(this);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-13470.
GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)
Posted by cn...@apache.org.
HADOOP-13470. GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23532716
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23532716
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23532716
Branch: refs/heads/HADOOP-13345
Commit: 23532716fcd3f7e5e20b8f9fc66188041638510a
Parents: 3808876
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Aug 16 16:30:43 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Aug 16 17:33:04 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/test/GenericTestUtils.java | 31 ++++++++------
.../hadoop/test/TestGenericTestUtils.java | 44 ++++++++++++++++++++
2 files changed, 63 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23532716/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 116a111..0b73cf5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -42,10 +42,12 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
import org.junit.Assume;
@@ -275,36 +277,41 @@ public abstract class GenericTestUtils {
private StringWriter sw = new StringWriter();
private WriterAppender appender;
private Logger logger;
-
+
public static LogCapturer captureLogs(Log l) {
Logger logger = ((Log4JLogger)l).getLogger();
- LogCapturer c = new LogCapturer(logger);
- return c;
+ return new LogCapturer(logger);
+ }
+
+ public static LogCapturer captureLogs(org.slf4j.Logger logger) {
+ return new LogCapturer(toLog4j(logger));
}
-
private LogCapturer(Logger logger) {
this.logger = logger;
- Layout layout = Logger.getRootLogger().getAppender("stdout").getLayout();
- WriterAppender wa = new WriterAppender(layout, sw);
- logger.addAppender(wa);
+ Appender defaultAppender = Logger.getRootLogger().getAppender("stdout");
+ if (defaultAppender == null) {
+ defaultAppender = Logger.getRootLogger().getAppender("console");
+ }
+ final Layout layout = (defaultAppender == null) ? new PatternLayout() :
+ defaultAppender.getLayout();
+ this.appender = new WriterAppender(layout, sw);
+ logger.addAppender(this.appender);
}
-
+
public String getOutput() {
return sw.toString();
}
-
+
public void stopCapturing() {
logger.removeAppender(appender);
-
}
public void clearOutput() {
sw.getBuffer().setLength(0);
}
}
-
-
+
/**
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23532716/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
index 8a7b5f6..86df5d5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
@@ -18,8 +18,16 @@
package org.apache.hadoop.test;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertTrue;
+
public class TestGenericTestUtils extends GenericTestUtils {
@Test
@@ -75,4 +83,40 @@ public class TestGenericTestUtils extends GenericTestUtils {
}
}
+ @Test(timeout = 10000)
+ public void testLogCapturer() {
+ final Log log = LogFactory.getLog(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(log);
+ final String infoMessage = "info message";
+ // test get output message
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ log.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
+ @Test(timeout = 10000)
+ public void testLogCapturerSlf4jLogger() {
+ final Logger logger = LoggerFactory.getLogger(TestGenericTestUtils.class);
+ LogCapturer logCapturer = LogCapturer.captureLogs(logger);
+ final String infoMessage = "info message";
+ // test get output message
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().endsWith(
+ String.format(infoMessage + "%n")));
+ // test clear output
+ logCapturer.clearOutput();
+ assertTrue(logCapturer.getOutput().isEmpty());
+ // test stop capturing
+ logCapturer.stopCapturing();
+ logger.info(infoMessage);
+ assertTrue(logCapturer.getOutput().isEmpty());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-13419. Fix javadoc warnings
by JDK8 in hadoop-common package. Contributed by Kai Sasaki.
Posted by cn...@apache.org.
HADOOP-13419. Fix javadoc warnings by JDK8 in hadoop-common package. Contributed by Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a446ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a446ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a446ba
Branch: refs/heads/HADOOP-13345
Commit: b8a446ba57d89c0896ec2d56dd919b0101e69f44
Parents: 4d4d95f
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Tue Aug 16 13:30:40 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Tue Aug 16 13:30:40 2016 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileContext.java | 4 +-
.../apache/hadoop/io/retry/package-info.java | 22 +++++++++
.../org/apache/hadoop/io/retry/package.html | 48 --------------------
.../org/apache/hadoop/ipc/package-info.java | 4 ++
.../java/org/apache/hadoop/ipc/package.html | 23 ----------
5 files changed, 28 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a446ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index e6a4cf4..f235773 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -303,7 +303,7 @@ public class FileContext {
*
* @throws UnsupportedFileSystemException If the file system for
* <code>absOrFqPath</code> is not supported.
- * @throws IOExcepton If the file system for <code>absOrFqPath</code> could
+ * @throws IOException If the file system for <code>absOrFqPath</code> could
* not be instantiated.
*/
protected AbstractFileSystem getFSofPath(final Path absOrFqPath)
@@ -2713,7 +2713,7 @@ public class FileContext {
/**
* Query the effective storage policy ID for the given file or directory.
*
- * @param src file or directory path.
+ * @param path file or directory path.
* @return storage policy for give file.
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a446ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package-info.java
index 693065f..089cf6f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package-info.java
@@ -15,6 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * A mechanism for selectively retrying methods that throw exceptions under
+ * certain circumstances.
+ * Typical usage is
+ * UnreliableImplementation unreliableImpl = new UnreliableImplementation();
+ * UnreliableInterface unreliable = (UnreliableInterface)
+ * RetryProxy.create(UnreliableInterface.class, unreliableImpl,
+ * RetryPolicies.retryUpToMaximumCountWithFixedSleep(4, 10,
+ * TimeUnit.SECONDS));
+ * unreliable.call();
+ *
+ * This will retry any method called on <code>unreliable</code> four times -
+ * in this case the <code>call()</code> method - sleeping 10 seconds between
+ * each retry. There are a number of
+ * {@link org.apache.hadoop.io.retry.RetryPolicies retry policies}
+ * available, or you can implement a custom one by implementing
+ * {@link org.apache.hadoop.io.retry.RetryPolicy}.
+ * It is also possible to specify retry policies on a
+ * {@link org.apache.hadoop.io.retry.RetryProxy#create(Class, Object, Map)
+ * per-method basis}.
+ */
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "MapReduce"})
@InterfaceStability.Evolving
package org.apache.hadoop.io.retry;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a446ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package.html b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package.html
deleted file mode 100644
index ae553fc..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/package.html
+++ /dev/null
@@ -1,48 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-
-<p>
-A mechanism for selectively retrying methods that throw exceptions under certain circumstances.
-</p>
-
-<p>
-Typical usage is
-</p>
-
-<pre>
-UnreliableImplementation unreliableImpl = new UnreliableImplementation();
-UnreliableInterface unreliable = (UnreliableInterface)
- RetryProxy.create(UnreliableInterface.class, unreliableImpl,
- RetryPolicies.retryUpToMaximumCountWithFixedSleep(4, 10, TimeUnit.SECONDS));
-unreliable.call();
-</pre>
-
-<p>
-This will retry any method called on <code>unreliable</code> four times - in this case the <code>call()</code>
-method - sleeping 10 seconds between
-each retry. There are a number of {@link org.apache.hadoop.io.retry.RetryPolicies retry policies}
-available, or you can implement a custom one by implementing {@link org.apache.hadoop.io.retry.RetryPolicy}.
-It is also possible to specify retry policies on a
-{@link org.apache.hadoop.io.retry.RetryProxy#create(Class, Object, Map) per-method basis}.
-</p>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a446ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java
index 525ef1f..cb35e93 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Tools to help define network clients and servers.
+ */
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "MapReduce"})
@InterfaceStability.Evolving
package org.apache.hadoop.ipc;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a446ba/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package.html b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package.html
deleted file mode 100644
index 3efd81a..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/package.html
+++ /dev/null
@@ -1,23 +0,0 @@
-<html>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<body>
-Tools to help define network clients and servers.
-</body>
-</html>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5455. Update Javadocs for
LinuxContainerExecutor. Contributed by Daniel Templeton.
Posted by cn...@apache.org.
YARN-5455. Update Javadocs for LinuxContainerExecutor. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f05ff7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f05ff7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f05ff7a
Branch: refs/heads/HADOOP-13345
Commit: 7f05ff7a4e654693eaaa216ee5fc6e24112e0e23
Parents: 2353271
Author: Varun Vasudev <vv...@apache.org>
Authored: Wed Aug 17 15:34:58 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Wed Aug 17 15:34:58 2016 +0530
----------------------------------------------------------------------
.../nodemanager/LinuxContainerExecutor.java | 80 ++++++++++-
.../runtime/DefaultLinuxContainerRuntime.java | 20 ++-
.../DelegatingLinuxContainerRuntime.java | 10 ++
.../runtime/DockerLinuxContainerRuntime.java | 143 ++++++++++++++++---
.../linux/runtime/LinuxContainerRuntime.java | 10 +-
.../runtime/ContainerRuntime.java | 40 +++++-
6 files changed, 267 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 8f5ee6b..2c27f6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -38,7 +38,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DefaultLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DelegatingLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
@@ -64,11 +66,36 @@ import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
-/** Container execution for Linux. Provides linux-specific localization
- * mechanisms, resource management via cgroups and can switch between multiple
- * container runtimes - e.g Standard "Process Tree", Docker etc
+/**
+ * <p>This class provides {@link Container} execution using a native
+ * {@code container-executor} binary. By using a helper written it native code,
+ * this class is able to do several things that the
+ * {@link DefaultContainerExecutor} cannot, such as execution of applications
+ * as the applications' owners, provide localization that takes advantage of
+ * mapping the application owner to a UID on the execution host, resource
+ * management through Linux CGROUPS, and Docker support.</p>
+ *
+ * <p>If {@code hadoop.security.authetication} is set to {@code simple},
+ * then the
+ * {@code yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users}
+ * property will determine whether the {@code LinuxContainerExecutor} runs
+ * processes as the application owner or as the default user, as set in the
+ * {@code yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user}
+ * property.</p>
+ *
+ * <p>The {@code LinuxContainerExecutor} will manage applications through an
+ * appropriate {@link LinuxContainerRuntime} instance. This class uses a
+ * {@link DelegatingLinuxContainerRuntime} instance, which will delegate calls
+ * to either a {@link DefaultLinuxContainerRuntime} instance or a
+ * {@link DockerLinuxContainerRuntime} instance, depending on the job's
+ * configuration.</p>
+ *
+ * @see LinuxContainerRuntime
+ * @see DelegatingLinuxContainerRuntime
+ * @see DefaultLinuxContainerRuntime
+ * @see DockerLinuxContainerRuntime
+ * @see DockerLinuxContainerRuntime#isDockerContainerRequested
*/
-
public class LinuxContainerExecutor extends ContainerExecutor {
private static final Log LOG = LogFactory
@@ -83,10 +110,18 @@ public class LinuxContainerExecutor extends ContainerExecutor {
private ResourceHandler resourceHandlerChain;
private LinuxContainerRuntime linuxContainerRuntime;
+ /**
+ * Default constructor to allow for creation through reflection.
+ */
public LinuxContainerExecutor() {
}
- // created primarily for testing
+ /**
+ * Create a LinuxContainerExecutor with a provided
+ * {@link LinuxContainerRuntime}. Used primarily for testing.
+ *
+ * @param linuxContainerRuntime the runtime to use
+ */
public LinuxContainerExecutor(LinuxContainerRuntime linuxContainerRuntime) {
this.linuxContainerRuntime = linuxContainerRuntime;
}
@@ -154,6 +189,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
}
+ /**
+ * Get the path to the {@code container-executor} binary. The path will
+ * be absolute.
+ *
+ * @param conf the {@link Configuration}
+ * @return the path to the {@code container-executor} binary
+ */
protected String getContainerExecutorExecutablePath(Configuration conf) {
String yarnHomeEnvVar =
System.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
@@ -166,6 +208,14 @@ public class LinuxContainerExecutor extends ContainerExecutor {
defaultPath);
}
+ /**
+ * Add a niceness level to the process that will be executed. Adds
+ * {@code -n <nice>} to the given command. The niceness level will be
+ * taken from the
+ * {@code yarn.nodemanager.container-executer.os.sched.prioity} property.
+ *
+ * @param command the command to which to add the niceness setting.
+ */
protected void addSchedPriorityCommand(List<String> command) {
if (containerSchedPriorityIsSet) {
command.addAll(Arrays.asList("nice", "-n",
@@ -294,6 +344,17 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
}
+ /**
+ * Set up the {@link ContainerLocalizer}.
+ *
+ * @param command the current ShellCommandExecutor command line
+ * @param user localization user
+ * @param appId localized app id
+ * @param locId localizer id
+ * @param nmAddr nodemanager address
+ * @param localDirs list of local dirs
+ * @see ContainerLocalizer#buildMainArgs
+ */
@VisibleForTesting
public void buildMainArgs(List<String> command, String user, String appId,
String locId, InetSocketAddress nmAddr, List<String> localDirs) {
@@ -594,6 +655,15 @@ public class LinuxContainerExecutor extends ContainerExecutor {
.build());
}
+ /**
+ * Mount a CGROUPS controller at the requested mount point and create
+ * a hierarchy for the NodeManager to manage.
+ *
+ * @param cgroupKVs a key-value pair of the form
+ * {@code controller=mount-path}
+ * @param hierarchy the top directory of the hierarchy for the NodeManager
+ * @throws IOException if there is a problem mounting the CGROUPS
+ */
public void mountCgroups(List<String> cgroupKVs, String hierarchy)
throws IOException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index e78f460..57e3cc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -31,20 +31,34 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import java.util.List;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+/**
+ * This class is a {@link ContainerRuntime} implementation that uses the
+ * native {@code container-executor} binary via a
+ * {@link PrivilegedOperationExecutor} instance to launch processes using the
+ * standard process model.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
- private static final Log LOG = LogFactory
- .getLog(DefaultLinuxContainerRuntime.class);
- private Configuration conf;
+ private static final Log LOG =
+ LogFactory.getLog(DefaultLinuxContainerRuntime.class);
private final PrivilegedOperationExecutor privilegedOperationExecutor;
+ private Configuration conf;
+ /**
+ * Create an instance using the given {@link PrivilegedOperationExecutor}
+ * instance for performing operations.
+ *
+ * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
+ * instance
+ */
public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor
privilegedOperationExecutor) {
this.privilegedOperationExecutor = privilegedOperationExecutor;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 975bb9b..faa7522 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -28,10 +28,20 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
import java.util.Map;
+/**
+ * This class is a {@link ContainerRuntime} implementation that delegates all
+ * operations to either a {@link DefaultLinuxContainerRuntime} instance or a
+ * {@link DockerLinuxContainerRuntime} instance, depending on whether the
+ * {@link DockerLinuxContainerRuntime} instance believes the operation to be
+ * requesting a Docker container.
+ *
+ * @see DockerLinuxContainerRuntime#isDockerContainerRequested
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index dc56ab0..be17af9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
@@ -58,6 +59,68 @@ import java.util.Set;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
+/**
+ * <p>This class is a {@link ContainerRuntime} implementation that uses the
+ * native {@code container-executor} binary via a
+ * {@link PrivilegedOperationExecutor} instance to launch processes inside
+ * Docker containers.</p>
+ *
+ * <p>The following environment variables are used to configure the Docker
+ * engine:</p>
+ *
+ * <ul>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_TYPE} ultimately determines whether a
+ * Docker container will be used. If the value is {@code docker}, a Docker
+ * container will be used. Otherwise a regular process tree container will
+ * be used. This environment variable is checked by the
+ * {@link #isDockerContainerRequested} method, which is called by the
+ * {@link DelegatingLinuxContainerRuntime}.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_IMAGE} names which image
+ * will be used to launch the Docker container.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_IMAGE_FILE} is currently ignored.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE} controls
+ * whether the Docker container's default command is overridden. When set
+ * to {@code true}, the Docker container's command will be
+ * {@code bash <path_to_launch_script>}. When unset or set to {@code false}
+ * the Docker container's default command is used.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK} sets the
+ * network type to be used by the Docker container. It must be a valid
+ * value as determined by the
+ * {@code yarn.nodemanager.runtime.linux.docker.allowed-container-networks}
+ * property.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
+ * controls whether the Docker container is a privileged container. In order
+ * to use privileged containers, the
+ * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed}
+ * property must be set to {@code true}, and the application owner must
+ * appear in the value of the
+ * {@code yarn.nodemanager.runtime.linux.docker.privileged-containers.acl}
+ * property. If this environment variable is set to {@code true}, a
+ * privileged Docker container will be used if allowed. No other value is
+ * allowed, so the environment variable should be left unset rather than
+ * setting it to false.
+ * </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS} adds
+ * additional volume mounts to the Docker container. The value of the
+ * environment variable should be a comma-separated list of mounts.
+ * All such mounts must be given as {@code source:dest}, where the
+ * source is an absolute path that is not a symlink and that points to a
+ * localized resource.
+ * </li>
+ * </ul>
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@@ -90,6 +153,15 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private CGroupsHandler cGroupsHandler;
private AccessControlList privilegedContainersAcl;
+ /**
+ * Return whether the given environment variables indicate that the operation
+ * is requesting a Docker container. If the environment contains a key
+ * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
+ * this method will return true. Otherwise it will return false.
+ *
+ * @param env the environment variable settings for the operation
+ * @return whether a Docker container is requested
+ */
public static boolean isDockerContainerRequested(
Map<String, String> env) {
if (env == null) {
@@ -101,13 +173,28 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return type != null && type.equals("docker");
}
+ /**
+ * Create an instance using the given {@link PrivilegedOperationExecutor}
+ * instance for performing operations.
+ *
+ * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
+ * instance
+ */
public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
privilegedOperationExecutor) {
- this(privilegedOperationExecutor, ResourceHandlerModule
- .getCGroupsHandler());
+ this(privilegedOperationExecutor,
+ ResourceHandlerModule.getCGroupsHandler());
}
- //A constructor with an injected cGroupsHandler primarily used for testing.
+ /**
+ * Create an instance using the given {@link PrivilegedOperationExecutor}
+ * instance for performing operations and the given {@link CGroupsHandler}
+ * instance. This constructor is intended for use in testing.
+ *
+ * @param privilegedOperationExecutor the {@link PrivilegedOperationExecutor}
+ * instance
+ * @param cGroupsHandler the {@link CGroupsHandler} instance
+ */
@VisibleForTesting
public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
privilegedOperationExecutor, CGroupsHandler cGroupsHandler) {
@@ -155,7 +242,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
-
}
private void validateContainerNetworkType(String network)
@@ -170,9 +256,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
throw new ContainerExecutionException(msg);
}
- public void addCGroupParentIfRequired(String resourcesOptions,
- String containerIdStr, DockerRunCommand runCommand)
- throws ContainerExecutionException {
+ /**
+ * If CGROUPS in enabled and not set to none, then set the CGROUP parent for
+ * the command instance.
+ *
+ * @param resourcesOptions the resource options to check for "cgroups=none"
+ * @param containerIdStr the container ID
+ * @param runCommand the command to set with the CGROUP parent
+ */
+ @VisibleForTesting
+ protected void addCGroupParentIfRequired(String resourcesOptions,
+ String containerIdStr, DockerRunCommand runCommand) {
if (cGroupsHandler == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("cGroupsHandler is null. cgroups are not in use. nothing to"
@@ -181,9 +275,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return;
}
- if (resourcesOptions.equals(
- (PrivilegedOperation.CGROUP_ARG_PREFIX + PrivilegedOperation
- .CGROUP_ARG_NO_TASKS))) {
+ if (resourcesOptions.equals(PrivilegedOperation.CGROUP_ARG_PREFIX
+ + PrivilegedOperation.CGROUP_ARG_NO_TASKS)) {
if (LOG.isDebugEnabled()) {
LOG.debug("no resource restrictions specified. not using docker's "
+ "cgroup options");
@@ -193,8 +286,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOG.debug("using docker's cgroups options");
}
- String cGroupPath = "/" + cGroupsHandler.getRelativePathForCGroup(
- containerIdStr);
+ String cGroupPath = "/"
+ + cGroupsHandler.getRelativePathForCGroup(containerIdStr);
if (LOG.isDebugEnabled()) {
LOG.debug("using cgroup parent: " + cGroupPath);
@@ -204,14 +297,25 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ /**
+ * Return whether the YARN container is allowed to run in a privileged
+ * Docker container. For a privileged container to be allowed all of the
+ * following three conditions must be satisfied:
+ *
+ * <ol>
+ * <li>Submitting user must request for a privileged container</li>
+ * <li>Privileged containers must be enabled on the cluster</li>
+ * <li>Submitting user must be white-listed to run a privileged
+ * container</li>
+ * </ol>
+ *
+ * @param container the target YARN container
+ * @return whether privileged container execution is allowed
+ * @throws ContainerExecutionException if privileged container execution
+ * is requested but is not allowed
+ */
private boolean allowPrivilegedContainerExecution(Container container)
throws ContainerExecutionException {
- //For a privileged container to be run all of the following three conditions
- // must be satisfied:
- //1) Submitting user must request for a privileged container
- //2) Privileged containers must be enabled on the cluster
- //3) Submitting user must be whitelisted to run a privileged container
-
Map<String, String> environment = container.getLaunchContext()
.getEnvironment();
String runPrivilegedContainerEnvVar = environment
@@ -379,7 +483,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
- Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
+ Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(
NM_PRIVATE_CONTAINER_SCRIPT_PATH);
String disableOverride = environment.get(
@@ -487,6 +591,5 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@Override
public void reapContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
index 38aea9d..cd7a2f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntime.java
@@ -26,13 +26,21 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
-/** Linux-specific container runtime implementations must implement this
+/**
+ * Linux-specific container runtime implementations must implement this
* interface.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public interface LinuxContainerRuntime extends ContainerRuntime {
+ /**
+ * Initialize the runtime.
+ *
+ * @param conf the {@link Configuration} to use
+ * @throws ContainerExecutionException if an error occurs while initializing
+ * the runtime
+ */
void initialize(Configuration conf) throws ContainerExecutionException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f05ff7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index e05f3fc..653a4c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -23,28 +23,54 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-/** An abstraction for various container runtime implementations. Examples
- * include Process Tree, Docker, Appc runtimes etc., These implementations
+/**
+ * An abstraction for various container runtime implementations. Examples
+ * include Process Tree, Docker, Appc runtimes etc. These implementations
* are meant for low-level OS container support - dependencies on
- * higher-level nodemananger constructs should be avoided.
+ * higher-level node mananger constructs should be avoided.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public interface ContainerRuntime {
- /** Prepare a container to be ready for launch */
+ /**
+ * Prepare a container to be ready for launch.
+ *
+ * @param ctx the {@link ContainerRuntimeContext}
+ * @throws ContainerExecutionException if an error occurs while preparing
+ * the container
+ */
void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException;
- /** Launch a container. */
+ /**
+ * Launch a container.
+ *
+ * @param ctx the {@link ContainerRuntimeContext}
+ * @throws ContainerExecutionException if an error occurs while launching
+ * the container
+ */
void launchContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException;
- /** Signal a container - request to terminate, status check etc., */
+ /**
+ * Signal a container. Signals may be a request to terminate, a status check,
+ * etc.
+ *
+ * @param ctx the {@link ContainerRuntimeContext}
+ * @throws ContainerExecutionException if an error occurs while signaling
+ * the container
+ */
void signalContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException;
- /** Any container cleanup that may be required. */
+ /**
+ * Perform any container cleanup that may be required.
+ *
+ * @param ctx the {@link ContainerRuntimeContext}
+ * @throws ContainerExecutionException if an error occurs while reaping
+ * the container
+ */
void reapContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException;
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-5514. Clarify
DecommissionType.FORCEFUL comment (Vrushali C via Varun Saxena)
Posted by cn...@apache.org.
YARN-5514. Clarify DecommissionType.FORCEFUL comment (Vrushali C via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ffe1fff5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ffe1fff5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ffe1fff5
Branch: refs/heads/HADOOP-13345
Commit: ffe1fff5262bb1d77f68059f12ae43f9849b530f
Parents: b8a446b
Author: Varun Saxena <va...@apache.org>
Authored: Tue Aug 16 14:05:41 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 16 14:05:41 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/api/records/DecommissionType.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ffe1fff5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DecommissionType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DecommissionType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DecommissionType.java
index 988fd51..ba5609d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DecommissionType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/DecommissionType.java
@@ -17,13 +17,19 @@
*/
package org.apache.hadoop.yarn.api.records;
+/**
+ * Specifies the different types of decommissioning of nodes.
+ */
public enum DecommissionType {
- /** Decomissioning nodes in normal way **/
+ /** Decomissioning nodes in normal way. **/
NORMAL,
- /** Graceful decommissioning of nodes **/
+ /** Graceful decommissioning of nodes. **/
GRACEFUL,
- /** Forceful decommissioning of nodes which are already in progress **/
+ /**
+ * Forceful decommissioning of nodes whose decommissioning is already in
+ * progress.
+ **/
FORCEFUL
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: Revert "HADOOP-13470.
GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)"
Posted by cn...@apache.org.
Revert "HADOOP-13470. GenericTestUtils$LogCapturer is flaky. (Contributed by Mingliang Liu)"
This reverts commit 9336a0495f99cd3fbc7ecef452eb37cfbaf57440.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27a6e09c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27a6e09c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27a6e09c
Branch: refs/heads/HADOOP-13345
Commit: 27a6e09c4e22b9b5fee4e8ced7321eed92d566a4
Parents: 4b689e7
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Aug 16 16:25:37 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Aug 16 16:25:37 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/test/GenericTestUtils.java | 25 +++++------
.../hadoop/test/TestGenericTestUtils.java | 44 --------------------
2 files changed, 13 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a6e09c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 6b5135c..116a111 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -42,10 +42,10 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
import org.junit.Assume;
@@ -275,35 +275,36 @@ public abstract class GenericTestUtils {
private StringWriter sw = new StringWriter();
private WriterAppender appender;
private Logger logger;
-
+
public static LogCapturer captureLogs(Log l) {
Logger logger = ((Log4JLogger)l).getLogger();
- return new LogCapturer(logger);
- }
-
- public static LogCapturer captureLogs(org.slf4j.Logger logger) {
- return new LogCapturer(toLog4j(logger));
+ LogCapturer c = new LogCapturer(logger);
+ return c;
}
+
private LogCapturer(Logger logger) {
this.logger = logger;
- this.appender = new WriterAppender(new PatternLayout(), sw);
- logger.addAppender(appender);
+ Layout layout = Logger.getRootLogger().getAppender("stdout").getLayout();
+ WriterAppender wa = new WriterAppender(layout, sw);
+ logger.addAppender(wa);
}
-
+
public String getOutput() {
return sw.toString();
}
-
+
public void stopCapturing() {
logger.removeAppender(appender);
+
}
public void clearOutput() {
sw.getBuffer().setLength(0);
}
}
-
+
+
/**
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27a6e09c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
index 86df5d5..8a7b5f6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestGenericTestUtils.java
@@ -18,16 +18,8 @@
package org.apache.hadoop.test;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.junit.Assert.assertTrue;
-
public class TestGenericTestUtils extends GenericTestUtils {
@Test
@@ -83,40 +75,4 @@ public class TestGenericTestUtils extends GenericTestUtils {
}
}
- @Test(timeout = 10000)
- public void testLogCapturer() {
- final Log log = LogFactory.getLog(TestGenericTestUtils.class);
- LogCapturer logCapturer = LogCapturer.captureLogs(log);
- final String infoMessage = "info message";
- // test get output message
- log.info(infoMessage);
- assertTrue(logCapturer.getOutput().endsWith(
- String.format(infoMessage + "%n")));
- // test clear output
- logCapturer.clearOutput();
- assertTrue(logCapturer.getOutput().isEmpty());
- // test stop capturing
- logCapturer.stopCapturing();
- log.info(infoMessage);
- assertTrue(logCapturer.getOutput().isEmpty());
- }
-
- @Test(timeout = 10000)
- public void testLogCapturerSlf4jLogger() {
- final Logger logger = LoggerFactory.getLogger(TestGenericTestUtils.class);
- LogCapturer logCapturer = LogCapturer.captureLogs(logger);
- final String infoMessage = "info message";
- // test get output message
- logger.info(infoMessage);
- assertTrue(logCapturer.getOutput().endsWith(
- String.format(infoMessage + "%n")));
- // test clear output
- logCapturer.clearOutput();
- assertTrue(logCapturer.getOutput().isEmpty());
- // test stop capturing
- logCapturer.stopCapturing();
- logger.info(infoMessage);
- assertTrue(logCapturer.getOutput().isEmpty());
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-13513. Java 1.7 support for
org.apache.hadoop.fs.azure testcases. Contributed by Tibor Kiss.
Posted by cn...@apache.org.
HADOOP-13513. Java 1.7 support for org.apache.hadoop.fs.azure testcases. Contributed by Tibor Kiss.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae4db254
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae4db254
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae4db254
Branch: refs/heads/HADOOP-13345
Commit: ae4db2544346370404826d5b55b2678f5f92fe1f
Parents: 0da69c3
Author: Chris Nauroth <cn...@apache.org>
Authored: Thu Aug 18 10:44:36 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Thu Aug 18 10:44:36 2016 -0700
----------------------------------------------------------------------
.../test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae4db254/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
index f0df38b..58d278a 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AbstractWasbTestBase.java
@@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory;
* to run those tests. The returned account might integrate with Azure Storage
* directly or it might be a mock implementation.
*/
-abstract class AbstractWasbTestBase {
+public abstract class AbstractWasbTestBase {
protected static final Logger LOG =
LoggerFactory.getLogger(AbstractWasbTestBase.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-11588. Benchmark framework
and test for erasure coders. Contributed by Rui Li
Posted by cn...@apache.org.
HADOOP-11588. Benchmark framework and test for erasure coders. Contributed by Rui Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fbb57fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fbb57fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fbb57fb
Branch: refs/heads/HADOOP-13345
Commit: 8fbb57fbd903a838684fa87cf15767d13695e4ed
Parents: 5199db3
Author: Kai Zheng <ka...@intel.com>
Authored: Fri Aug 12 15:05:52 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Fri Aug 12 15:05:52 2016 +0800
----------------------------------------------------------------------
.../rawcoder/RSRawDecoderLegacy.java | 56 +--
.../rawcoder/RawErasureCoderBenchmark.java | 408 +++++++++++++++++++
.../rawcoder/TestRawErasureCoderBenchmark.java | 65 +++
3 files changed, 490 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fbb57fb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java
index 0183760..c8deec9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java
@@ -39,28 +39,6 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
private int[] errSignature;
private int[] primitivePower;
- /**
- * We need a set of reusable buffers either for the bytes array
- * decoding version or direct buffer decoding version. Normally not both.
- *
- * For output, in addition to the valid buffers from the caller
- * passed from above, we need to provide extra buffers for the internal
- * decoding implementation. For output, the caller should provide no more
- * than numParityUnits but at least one buffers. And the left buffers will be
- * borrowed from either bytesArrayBuffers, for the bytes array version.
- *
- */
- // Reused buffers for decoding with bytes arrays
- private byte[][] bytesArrayBuffers = new byte[getNumParityUnits()][];
- private byte[][] adjustedByteArrayOutputsParameter =
- new byte[getNumParityUnits()][];
- private int[] adjustedOutputOffsets = new int[getNumParityUnits()];
-
- // Reused buffers for decoding with direct ByteBuffers
- private ByteBuffer[] directBuffers = new ByteBuffer[getNumParityUnits()];
- private ByteBuffer[] adjustedDirectBufferOutputsParameter =
- new ByteBuffer[getNumParityUnits()];
-
public RSRawDecoderLegacy(ErasureCoderOptions coderOptions) {
super(coderOptions);
if (getNumAllUnits() >= RSUtil.GF.getFieldSize()) {
@@ -139,16 +117,14 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
* implementations, so we have to adjust them before calling doDecodeImpl.
*/
+ byte[][] bytesArrayBuffers = new byte[getNumParityUnits()][];
+ byte[][] adjustedByteArrayOutputsParameter =
+ new byte[getNumParityUnits()][];
+ int[] adjustedOutputOffsets = new int[getNumParityUnits()];
+
int[] erasedOrNotToReadIndexes =
CoderUtil.getNullIndexes(decodingState.inputs);
- // Prepare for adjustedOutputsParameter
-
- // First reset the positions needed this time
- for (int i = 0; i < erasedOrNotToReadIndexes.length; i++) {
- adjustedByteArrayOutputsParameter[i] = null;
- adjustedOutputOffsets[i] = 0;
- }
// Use the caller passed buffers in erasedIndexes positions
for (int outputIdx = 0, i = 0;
i < decodingState.erasedIndexes.length; i++) {
@@ -174,7 +150,8 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
for (int bufferIdx = 0, i = 0; i < erasedOrNotToReadIndexes.length; i++) {
if (adjustedByteArrayOutputsParameter[i] == null) {
adjustedByteArrayOutputsParameter[i] = CoderUtil.resetBuffer(
- checkGetBytesArrayBuffer(bufferIdx, dataLen), 0, dataLen);
+ checkGetBytesArrayBuffer(bytesArrayBuffers, bufferIdx, dataLen),
+ 0, dataLen);
adjustedOutputOffsets[i] = 0; // Always 0 for such temp output
bufferIdx++;
}
@@ -198,12 +175,10 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
int[] erasedOrNotToReadIndexes =
CoderUtil.getNullIndexes(decodingState.inputs);
- // Prepare for adjustedDirectBufferOutputsParameter
+ ByteBuffer[] directBuffers = new ByteBuffer[getNumParityUnits()];
+ ByteBuffer[] adjustedDirectBufferOutputsParameter =
+ new ByteBuffer[getNumParityUnits()];
- // First reset the positions needed this time
- for (int i = 0; i < erasedOrNotToReadIndexes.length; i++) {
- adjustedDirectBufferOutputsParameter[i] = null;
- }
// Use the caller passed buffers in erasedIndexes positions
for (int outputIdx = 0, i = 0;
i < decodingState.erasedIndexes.length; i++) {
@@ -225,7 +200,8 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
// Use shared buffers for other positions (not set yet)
for (int bufferIdx = 0, i = 0; i < erasedOrNotToReadIndexes.length; i++) {
if (adjustedDirectBufferOutputsParameter[i] == null) {
- ByteBuffer buffer = checkGetDirectBuffer(bufferIdx, dataLen);
+ ByteBuffer buffer = checkGetDirectBuffer(
+ directBuffers, bufferIdx, dataLen);
buffer.position(0);
buffer.limit(dataLen);
adjustedDirectBufferOutputsParameter[i] =
@@ -274,15 +250,17 @@ public class RSRawDecoderLegacy extends RawErasureDecoder {
numErasedParityUnits, numErasedDataUnits);
}
- private byte[] checkGetBytesArrayBuffer(int idx, int bufferLen) {
+ private static byte[] checkGetBytesArrayBuffer(byte[][] bytesArrayBuffers,
+ int idx, int bufferLen) {
if (bytesArrayBuffers[idx] == null ||
- bytesArrayBuffers[idx].length < bufferLen) {
+ bytesArrayBuffers[idx].length < bufferLen) {
bytesArrayBuffers[idx] = new byte[bufferLen];
}
return bytesArrayBuffers[idx];
}
- private ByteBuffer checkGetDirectBuffer(int idx, int bufferLen) {
+ private static ByteBuffer checkGetDirectBuffer(ByteBuffer[] directBuffers,
+ int idx, int bufferLen) {
if (directBuffers[idx] == null ||
directBuffers[idx].capacity() < bufferLen) {
directBuffers[idx] = ByteBuffer.allocateDirect(bufferLen);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fbb57fb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
new file mode 100644
index 0000000..4492e2f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
@@ -0,0 +1,408 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.util.StopWatch;
+
+import java.nio.ByteBuffer;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A benchmark tool to test the performance of different erasure coders.
+ * The tool launches multiple threads to encode/decode certain amount of data,
+ * and measures the total throughput. It only focuses on performance and doesn't
+ * validate correctness of the encoded/decoded results.
+ * User can specify the data size each thread processes, as well as the chunk
+ * size to use for the coder.
+ * Different coders are supported. User can specify the coder by a coder index.
+ * The coder is shared among all the threads.
+ */
+public final class RawErasureCoderBenchmark {
+
+ private RawErasureCoderBenchmark() {
+ // prevent instantiation
+ }
+
+ // target size of input data buffer
+ private static final int TARGET_BUFFER_SIZE_MB = 126;
+
+ private static final int MAX_CHUNK_SIZE =
+ TARGET_BUFFER_SIZE_MB / BenchData.NUM_DATA_UNITS * 1024;
+
+ private static final List<RawErasureCoderFactory> CODER_MAKERS =
+ Collections.unmodifiableList(
+ Arrays.asList(new DummyRawErasureCoderFactory(),
+ new RSRawErasureCoderFactoryLegacy(),
+ new RSRawErasureCoderFactory(),
+ new NativeRSRawErasureCoderFactory()));
+
+ enum CODER {
+ DUMMY_CODER("Dummy coder"),
+ LEGACY_RS_CODER("Legacy Reed-Solomon Java coder"),
+ RS_CODER("Reed-Solomon Java coder"),
+ ISAL_CODER("ISA-L coder");
+
+ private final String name;
+
+ CODER(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ static {
+ Preconditions.checkArgument(CODER_MAKERS.size() == CODER.values().length);
+ }
+
+ private static void printAvailableCoders() {
+ StringBuilder sb = new StringBuilder(
+ "Available coders with coderIndex:\n");
+ for (CODER coder : CODER.values()) {
+ sb.append(coder.ordinal()).append(":").append(coder).append("\n");
+ }
+ System.out.println(sb.toString());
+ }
+
+ private static void usage(String message) {
+ if (message != null) {
+ System.out.println(message);
+ }
+ System.out.println(
+ "Usage: RawErasureCoderBenchmark <encode/decode> <coderIndex> " +
+ "[numThreads] [dataSize-in-MB] [chunkSize-in-KB]");
+ printAvailableCoders();
+ System.exit(1);
+ }
+
+ public static void main(String[] args) throws Exception {
+ String opType = null;
+ int coderIndex = 0;
+ // default values
+ int dataSizeMB = 10240;
+ int chunkSizeKB = 1024;
+ int numThreads = 1;
+
+ if (args.length > 1) {
+ opType = args[0];
+ if (!"encode".equals(opType) && !"decode".equals(opType)) {
+ usage("Invalid type: should be either 'encode' or 'decode'");
+ }
+
+ try {
+ coderIndex = Integer.parseInt(args[1]);
+ if (coderIndex < 0 || coderIndex >= CODER.values().length) {
+ usage("Invalid coder index, should be [0-" +
+ (CODER.values().length - 1) + "]");
+ }
+ } catch (NumberFormatException e) {
+ usage("Malformed coder index, " + e.getMessage());
+ }
+ } else {
+ usage(null);
+ }
+
+ if (args.length > 2) {
+ try {
+ numThreads = Integer.parseInt(args[2]);
+ if (numThreads <= 0) {
+ usage("Invalid number of threads.");
+ }
+ } catch (NumberFormatException e) {
+ usage("Malformed number of threads, " + e.getMessage());
+ }
+ }
+
+ if (args.length > 3) {
+ try {
+ dataSizeMB = Integer.parseInt(args[3]);
+ if (dataSizeMB <= 0) {
+ usage("Invalid data size.");
+ }
+ } catch (NumberFormatException e) {
+ usage("Malformed data size, " + e.getMessage());
+ }
+ }
+
+ if (args.length > 4) {
+ try {
+ chunkSizeKB = Integer.parseInt(args[4]);
+ if (chunkSizeKB <= 0) {
+ usage("Chunk size should be positive.");
+ }
+ if (chunkSizeKB > MAX_CHUNK_SIZE) {
+ usage("Chunk size should be no larger than " + MAX_CHUNK_SIZE);
+ }
+ } catch (NumberFormatException e) {
+ usage("Malformed chunk size, " + e.getMessage());
+ }
+ }
+
+ performBench(opType, CODER.values()[coderIndex],
+ numThreads, dataSizeMB, chunkSizeKB);
+ }
+
+ /**
+ * Performs benchmark.
+ *
+ * @param opType The operation to perform. Can be encode or decode
+ * @param coder The coder to use
+ * @param numThreads Number of threads to launch concurrently
+ * @param dataSizeMB Total test data size in MB
+ * @param chunkSizeKB Chunk size in KB
+ */
+ public static void performBench(String opType, CODER coder,
+ int numThreads, int dataSizeMB, int chunkSizeKB) throws Exception {
+ BenchData.configure(dataSizeMB, chunkSizeKB);
+
+ RawErasureEncoder encoder = null;
+ RawErasureDecoder decoder = null;
+ ByteBuffer testData;
+ boolean isEncode = opType.equals("encode");
+
+ if (isEncode) {
+ encoder = getRawEncoder(coder.ordinal());
+ testData = genTestData(encoder.preferDirectBuffer(),
+ BenchData.bufferSizeKB);
+ } else {
+ decoder = getRawDecoder(coder.ordinal());
+ testData = genTestData(decoder.preferDirectBuffer(),
+ BenchData.bufferSizeKB);
+ }
+
+ ExecutorService executor = Executors.newFixedThreadPool(numThreads);
+ List<Future<Long>> futures = new ArrayList<>(numThreads);
+ StopWatch sw = new StopWatch().start();
+ for (int i = 0; i < numThreads; i++) {
+ futures.add(executor.submit(new BenchmarkCallable(isEncode,
+ encoder, decoder, testData.duplicate())));
+ }
+ List<Long> durations = new ArrayList<>(numThreads);
+ try {
+ for (Future<Long> future : futures) {
+ durations.add(future.get());
+ }
+ long duration = sw.now(TimeUnit.MILLISECONDS);
+ double totalDataSize = BenchData.totalDataSizeKB * numThreads / 1024.0;
+ DecimalFormat df = new DecimalFormat("#.##");
+ System.out.println(coder + " " + opType + " " +
+ df.format(totalDataSize) + "MB data, with chunk size " +
+ BenchData.chunkSize / 1024 + "KB");
+ System.out.println("Total time: " + df.format(duration / 1000.0) + " s.");
+ System.out.println("Total throughput: " + df.format(
+ totalDataSize / duration * 1000.0) + " MB/s");
+ printThreadStatistics(durations, df);
+ } catch (Exception e) {
+ System.out.println("Error waiting for thread to finish.");
+ e.printStackTrace();
+ throw e;
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private static RawErasureEncoder getRawEncoder(int index) {
+ RawErasureEncoder encoder =
+ CODER_MAKERS.get(index).createEncoder(BenchData.OPTIONS);
+ final boolean isDirect = encoder.preferDirectBuffer();
+ encoder.encode(
+ getBufferForInit(BenchData.NUM_DATA_UNITS, 1, isDirect),
+ getBufferForInit(BenchData.NUM_PARITY_UNITS, 1, isDirect));
+ return encoder;
+ }
+
+ private static RawErasureDecoder getRawDecoder(int index) {
+ RawErasureDecoder decoder =
+ CODER_MAKERS.get(index).createDecoder(BenchData.OPTIONS);
+ final boolean isDirect = decoder.preferDirectBuffer();
+ ByteBuffer[] inputs = getBufferForInit(
+ BenchData.NUM_ALL_UNITS, 1, isDirect);
+ for (int erasedIndex : BenchData.ERASED_INDEXES) {
+ inputs[erasedIndex] = null;
+ }
+ decoder.decode(inputs, BenchData.ERASED_INDEXES,
+ getBufferForInit(BenchData.ERASED_INDEXES.length, 1, isDirect));
+ return decoder;
+ }
+
+ private static ByteBuffer[] getBufferForInit(int numBuf,
+ int bufCap, boolean isDirect) {
+ ByteBuffer[] buffers = new ByteBuffer[numBuf];
+ for (int i = 0; i < buffers.length; i++) {
+ buffers[i] = isDirect ? ByteBuffer.allocateDirect(bufCap) :
+ ByteBuffer.allocate(bufCap);
+ }
+ return buffers;
+ }
+
+ private static void printThreadStatistics(
+ List<Long> durations, DecimalFormat df) {
+ Collections.sort(durations);
+ System.out.println("Threads statistics: ");
+ Double min = durations.get(0) / 1000.0;
+ Double max = durations.get(durations.size() - 1) / 1000.0;
+ Long sum = 0L;
+ for (Long duration : durations) {
+ sum += duration;
+ }
+ Double avg = sum.doubleValue() / durations.size() / 1000.0;
+ Double percentile = durations.get(
+ (int) Math.ceil(durations.size() * 0.9) - 1) / 1000.0;
+ System.out.println(durations.size() + " threads in total.");
+ System.out.println("Min: " + df.format(min) + " s, Max: " +
+ df.format(max) + " s, Avg: " + df.format(avg) +
+ " s, 90th Percentile: " + df.format(percentile) + " s.");
+ }
+
+ private static ByteBuffer genTestData(boolean useDirectBuffer, int sizeKB) {
+ Random random = new Random();
+ int bufferSize = sizeKB * 1024;
+ byte[] tmp = new byte[bufferSize];
+ random.nextBytes(tmp);
+ ByteBuffer data = useDirectBuffer ?
+ ByteBuffer.allocateDirect(bufferSize) :
+ ByteBuffer.allocate(bufferSize);
+ data.put(tmp);
+ data.flip();
+ return data;
+ }
+
+ private static class BenchData {
+ public static final ErasureCoderOptions OPTIONS =
+ new ErasureCoderOptions(6, 3);
+ public static final int NUM_DATA_UNITS = OPTIONS.getNumDataUnits();
+ public static final int NUM_PARITY_UNITS = OPTIONS.getNumParityUnits();
+ public static final int NUM_ALL_UNITS = OPTIONS.getNumAllUnits();
+ private static int chunkSize;
+ private static long totalDataSizeKB;
+ private static int bufferSizeKB;
+
+ private static final int[] ERASED_INDEXES = new int[]{6, 7, 8};
+ private final ByteBuffer[] inputs = new ByteBuffer[NUM_DATA_UNITS];
+ private ByteBuffer[] outputs = new ByteBuffer[NUM_PARITY_UNITS];
+ private ByteBuffer[] decodeInputs = new ByteBuffer[NUM_ALL_UNITS];
+
+ public static void configure(int dataSizeMB, int chunkSizeKB) {
+ chunkSize = chunkSizeKB * 1024;
+ // buffer size needs to be a multiple of (numDataUnits * chunkSize)
+ int round = (int) Math.round(
+ TARGET_BUFFER_SIZE_MB * 1024.0 / NUM_DATA_UNITS / chunkSizeKB);
+ Preconditions.checkArgument(round > 0);
+ bufferSizeKB = NUM_DATA_UNITS * chunkSizeKB * round;
+ System.out.println("Using " + bufferSizeKB / 1024 + "MB buffer.");
+
+ round = (int) Math.round(
+ (dataSizeMB * 1024.0) / bufferSizeKB);
+ if (round == 0) {
+ round = 1;
+ }
+ totalDataSizeKB = round * bufferSizeKB;
+ }
+
+ public BenchData(boolean useDirectBuffer) {
+ for (int i = 0; i < outputs.length; i++) {
+ outputs[i] = useDirectBuffer ? ByteBuffer.allocateDirect(chunkSize) :
+ ByteBuffer.allocate(chunkSize);
+ }
+ }
+
+ public void prepareDecInput() {
+ System.arraycopy(inputs, 0, decodeInputs, 0, NUM_DATA_UNITS);
+ }
+
+ public void encode(RawErasureEncoder encoder) {
+ encoder.encode(inputs, outputs);
+ }
+
+ public void decode(RawErasureDecoder decoder) {
+ decoder.decode(decodeInputs, ERASED_INDEXES, outputs);
+ }
+ }
+
+ private static class BenchmarkCallable implements Callable<Long> {
+ private final boolean isEncode;
+ private final RawErasureEncoder encoder;
+ private final RawErasureDecoder decoder;
+ private final BenchData benchData;
+ private final ByteBuffer testData;
+
+ public BenchmarkCallable(boolean isEncode, RawErasureEncoder encoder,
+ RawErasureDecoder decoder, ByteBuffer testData) {
+ if (isEncode) {
+ Preconditions.checkArgument(encoder != null);
+ this.encoder = encoder;
+ this.decoder = null;
+ benchData = new BenchData(encoder.preferDirectBuffer());
+ } else {
+ Preconditions.checkArgument(decoder != null);
+ this.decoder = decoder;
+ this.encoder = null;
+ benchData = new BenchData(decoder.preferDirectBuffer());
+ }
+ this.isEncode = isEncode;
+ this.testData = testData;
+ }
+
+ @Override
+ public Long call() throws Exception {
+ long rounds = BenchData.totalDataSizeKB / BenchData.bufferSizeKB;
+
+ StopWatch sw = new StopWatch().start();
+ for (long i = 0; i < rounds; i++) {
+ while (testData.remaining() > 0) {
+ for (ByteBuffer output : benchData.outputs) {
+ output.clear();
+ }
+
+ for (int j = 0; j < benchData.inputs.length; j++) {
+ benchData.inputs[j] = testData.duplicate();
+ benchData.inputs[j].limit(
+ testData.position() + BenchData.chunkSize);
+ benchData.inputs[j] = benchData.inputs[j].slice();
+ testData.position(testData.position() + BenchData.chunkSize);
+ }
+
+ if (isEncode) {
+ benchData.encode(encoder);
+ } else {
+ benchData.prepareDecInput();
+ benchData.decode(decoder);
+ }
+ }
+ testData.clear();
+ }
+ return sw.now(TimeUnit.MILLISECONDS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fbb57fb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java
new file mode 100644
index 0000000..3ba0260
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawErasureCoderBenchmark.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
+import org.junit.Assume;
+import org.junit.Test;
+
+/**
+ * Tests for the raw erasure coder benchmark tool.
+ */
+public class TestRawErasureCoderBenchmark {
+
+ @Test
+ public void testDummyCoder() throws Exception {
+ // Dummy coder
+ RawErasureCoderBenchmark.performBench("encode",
+ RawErasureCoderBenchmark.CODER.DUMMY_CODER, 2, 100, 1024);
+ RawErasureCoderBenchmark.performBench("decode",
+ RawErasureCoderBenchmark.CODER.DUMMY_CODER, 5, 150, 100);
+ }
+
+ @Test
+ public void testLegacyRSCoder() throws Exception {
+ // Legacy RS Java coder
+ RawErasureCoderBenchmark.performBench("encode",
+ RawErasureCoderBenchmark.CODER.LEGACY_RS_CODER, 2, 80, 200);
+ RawErasureCoderBenchmark.performBench("decode",
+ RawErasureCoderBenchmark.CODER.LEGACY_RS_CODER, 5, 300, 350);
+ }
+
+ @Test
+ public void testRSCoder() throws Exception {
+ // RS Java coder
+ RawErasureCoderBenchmark.performBench("encode",
+ RawErasureCoderBenchmark.CODER.RS_CODER, 3, 200, 200);
+ RawErasureCoderBenchmark.performBench("decode",
+ RawErasureCoderBenchmark.CODER.RS_CODER, 4, 135, 20);
+ }
+
+ @Test
+ public void testISALCoder() throws Exception {
+ Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
+ // ISA-L coder
+ RawErasureCoderBenchmark.performBench("encode",
+ RawErasureCoderBenchmark.CODER.ISAL_CODER, 5, 300, 64);
+ RawErasureCoderBenchmark.performBench("decode",
+ RawErasureCoderBenchmark.CODER.ISAL_CODER, 6, 200, 128);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-10724. Document caller
context config keys. (Contributed by Mingliang Liu)
Posted by cn...@apache.org.
HDFS-10724. Document caller context config keys. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bcbef39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bcbef39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bcbef39
Branch: refs/heads/HADOOP-13345
Commit: 4bcbef39f7ca07601092919a7f2bea531a2dfa07
Parents: 12ad63d
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:20:33 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:20:33 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 28 ++++++++++++++++++++
1 file changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bcbef39/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 992b464..f9c3f72 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2325,4 +2325,32 @@
<value>org.apache.hadoop.fs.adl.Adl</value>
</property>
+ <property>
+ <name>hadoop.caller.context.enabled</name>
+ <value>false</value>
+ <description>When the feature is enabled, additional fields are written into
+ name-node audit log records for auditing coarse granularity operations.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.max.size</name>
+ <value>128</value>
+ <description>The maximum bytes a caller context string can have. If the
+ passed caller context is longer than this maximum bytes, client will
+ truncate it before sending to server. Note that the server may have a
+ different maximum size, and will truncate the caller context to the
+ maximum size it allows.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.caller.context.signature.max.size</name>
+ <value>40</value>
+ <description>
+ The caller's signature (optional) is for offline validation. If the
+ signature exceeds the maximum allowed bytes in server, the caller context
+ will be abandoned, in which case the caller context will not be recorded
+ in audit logs.
+ </description>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-4702. FairScheduler: Allow
setting maxResources for ad hoc queues. (Daniel Templeton via kasha)
Posted by cn...@apache.org.
YARN-4702. FairScheduler: Allow setting maxResources for ad hoc queues. (Daniel Templeton via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20f0eb87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20f0eb87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20f0eb87
Branch: refs/heads/HADOOP-13345
Commit: 20f0eb871c57cc4c5a6d19aae0e3745b6175509b
Parents: ca13e79
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Aug 17 17:40:20 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Aug 17 17:40:20 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 87 ++++++---
.../fair/AllocationFileLoaderService.java | 92 +++++----
.../scheduler/fair/FairScheduler.java | 31 +++
.../scheduler/fair/QueueManager.java | 162 ++++++++++++----
.../fair/TestAllocationFileLoaderService.java | 43 ++++-
.../scheduler/fair/TestFairScheduler.java | 187 ++++++++++++++++++-
.../scheduler/fair/TestQueueManager.java | 166 +++++++++++++++-
.../src/site/markdown/FairScheduler.md | 3 +
8 files changed, 658 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index f984fef..5cf110f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -46,6 +46,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
// Maximum amount of resources per queue
@VisibleForTesting
final Map<String, Resource> maxQueueResources;
+ // Maximum amount of resources for each queue's ad hoc children
+ private final Map<String, Resource> maxChildQueueResources;
// Sharing weights for each queue
private final Map<String, ResourceWeights> queueWeights;
@@ -107,6 +109,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
public AllocationConfiguration(Map<String, Resource> minQueueResources,
Map<String, Resource> maxQueueResources,
+ Map<String, Resource> maxChildQueueResources,
Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
Map<String, ResourceWeights> queueWeights,
Map<String, Float> queueMaxAMShares, int userMaxAppsDefault,
@@ -126,6 +129,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
Set<String> nonPreemptableQueues) {
this.minQueueResources = minQueueResources;
this.maxQueueResources = maxQueueResources;
+ this.maxChildQueueResources = maxChildQueueResources;
this.queueMaxApps = queueMaxApps;
this.userMaxApps = userMaxApps;
this.queueMaxAMShares = queueMaxAMShares;
@@ -149,31 +153,32 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
}
public AllocationConfiguration(Configuration conf) {
- minQueueResources = new HashMap<String, Resource>();
- maxQueueResources = new HashMap<String, Resource>();
- queueWeights = new HashMap<String, ResourceWeights>();
- queueMaxApps = new HashMap<String, Integer>();
- userMaxApps = new HashMap<String, Integer>();
- queueMaxAMShares = new HashMap<String, Float>();
+ minQueueResources = new HashMap<>();
+ maxChildQueueResources = new HashMap<>();
+ maxQueueResources = new HashMap<>();
+ queueWeights = new HashMap<>();
+ queueMaxApps = new HashMap<>();
+ userMaxApps = new HashMap<>();
+ queueMaxAMShares = new HashMap<>();
userMaxAppsDefault = Integer.MAX_VALUE;
queueMaxAppsDefault = Integer.MAX_VALUE;
queueMaxResourcesDefault = Resources.unbounded();
queueMaxAMShareDefault = 0.5f;
- queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
- resAcls = new HashMap<String, Map<ReservationACL, AccessControlList>>();
- minSharePreemptionTimeouts = new HashMap<String, Long>();
- fairSharePreemptionTimeouts = new HashMap<String, Long>();
- fairSharePreemptionThresholds = new HashMap<String, Float>();
- schedulingPolicies = new HashMap<String, SchedulingPolicy>();
+ queueAcls = new HashMap<>();
+ resAcls = new HashMap<>();
+ minSharePreemptionTimeouts = new HashMap<>();
+ fairSharePreemptionTimeouts = new HashMap<>();
+ fairSharePreemptionThresholds = new HashMap<>();
+ schedulingPolicies = new HashMap<>();
defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
reservableQueues = new HashSet<>();
- configuredQueues = new HashMap<FSQueueType, Set<String>>();
+ configuredQueues = new HashMap<>();
for (FSQueueType queueType : FSQueueType.values()) {
- configuredQueues.put(queueType, new HashSet<String>());
+ configuredQueues.put(queueType, new HashSet<>());
}
- placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
- configuredQueues);
- nonPreemptableQueues = new HashSet<String>();
+ placementPolicy =
+ QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
+ nonPreemptableQueues = new HashSet<>();
}
/**
@@ -263,7 +268,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
/**
* Get the minimum resource allocation for the given queue.
- * @return the cap set on this queue, or 0 if not set.
+ *
+ * @param queue the target queue's name
+ * @return the min allocation on this queue or {@link Resources#none}
+ * if not set
*/
public Resource getMinResources(String queue) {
Resource minQueueResource = minQueueResources.get(queue);
@@ -271,14 +279,26 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
}
/**
- * Get the maximum resource allocation for the given queue.
- * @return the cap set on this queue, or Integer.MAX_VALUE if not set.
+ * Set the maximum resource allocation for the given queue.
+ *
+ * @param queue the target queue
+ * @param maxResource the maximum resource allocation
*/
+ void setMaxResources(String queue, Resource maxResource) {
+ maxQueueResources.put(queue, maxResource);
+ }
- public Resource getMaxResources(String queueName) {
- Resource maxQueueResource = maxQueueResources.get(queueName);
+ /**
+ * Get the maximum resource allocation for the given queue. If the max in not
+ * set, return the larger of the min and the default max.
+ *
+ * @param queue the target queue's name
+ * @return the max allocation on this queue
+ */
+ public Resource getMaxResources(String queue) {
+ Resource maxQueueResource = maxQueueResources.get(queue);
if (maxQueueResource == null) {
- Resource minQueueResource = minQueueResources.get(queueName);
+ Resource minQueueResource = minQueueResources.get(queue);
if (minQueueResource != null &&
Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(),
minQueueResource, queueMaxResourcesDefault)) {
@@ -291,6 +311,27 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
}
}
+ /**
+ * Get the maximum resource allocation for children of the given queue.
+ *
+ * @param queue the target queue's name
+ * @return the max allocation on this queue or null if not set
+ */
+ public Resource getMaxChildResources(String queue) {
+ return maxChildQueueResources.get(queue);
+ }
+
+ /**
+ * Set the maximum resource allocation for the children of the given queue.
+ * Use of this method is primarily intended for testing purposes.
+ *
+ * @param queue the target queue
+ * @param maxResource the maximum resource allocation
+ */
+ void setMaxChildResources(String queue, Resource maxResource) {
+ maxChildQueueResources.put(queue, maxResource);
+ }
+
public boolean hasAccess(String queueName, QueueACL acl,
UserGroupInformation user) {
int lastPeriodIndex = queueName.length();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index fab536d..ee71981 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -213,23 +213,22 @@ public class AllocationFileLoaderService extends AbstractService {
LOG.info("Loading allocation file " + allocFile);
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
- Map<String, Resource> minQueueResources = new HashMap<String, Resource>();
- Map<String, Resource> maxQueueResources = new HashMap<String, Resource>();
- Map<String, Integer> queueMaxApps = new HashMap<String, Integer>();
- Map<String, Integer> userMaxApps = new HashMap<String, Integer>();
- Map<String, Float> queueMaxAMShares = new HashMap<String, Float>();
- Map<String, ResourceWeights> queueWeights = new HashMap<String, ResourceWeights>();
- Map<String, SchedulingPolicy> queuePolicies = new HashMap<String, SchedulingPolicy>();
- Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>();
- Map<String, Long> fairSharePreemptionTimeouts = new HashMap<String, Long>();
- Map<String, Float> fairSharePreemptionThresholds =
- new HashMap<String, Float>();
- Map<String, Map<QueueACL, AccessControlList>> queueAcls =
- new HashMap<String, Map<QueueACL, AccessControlList>>();
+ Map<String, Resource> minQueueResources = new HashMap<>();
+ Map<String, Resource> maxQueueResources = new HashMap<>();
+ Map<String, Resource> maxChildQueueResources = new HashMap<>();
+ Map<String, Integer> queueMaxApps = new HashMap<>();
+ Map<String, Integer> userMaxApps = new HashMap<>();
+ Map<String, Float> queueMaxAMShares = new HashMap<>();
+ Map<String, ResourceWeights> queueWeights = new HashMap<>();
+ Map<String, SchedulingPolicy> queuePolicies = new HashMap<>();
+ Map<String, Long> minSharePreemptionTimeouts = new HashMap<>();
+ Map<String, Long> fairSharePreemptionTimeouts = new HashMap<>();
+ Map<String, Float> fairSharePreemptionThresholds = new HashMap<>();
+ Map<String, Map<QueueACL, AccessControlList>> queueAcls = new HashMap<>();
Map<String, Map<ReservationACL, AccessControlList>> reservationAcls =
- new HashMap<String, Map<ReservationACL, AccessControlList>>();
- Set<String> reservableQueues = new HashSet<String>();
- Set<String> nonPreemptableQueues = new HashSet<String>();
+ new HashMap<>();
+ Set<String> reservableQueues = new HashSet<>();
+ Set<String> nonPreemptableQueues = new HashSet<>();
int userMaxAppsDefault = Integer.MAX_VALUE;
int queueMaxAppsDefault = Integer.MAX_VALUE;
Resource queueMaxResourcesDefault = Resources.unbounded();
@@ -250,8 +249,8 @@ public class AllocationFileLoaderService extends AbstractService {
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
// and also for making queue placement decisions(QueuePlacementRule.java).
- Map<FSQueueType, Set<String>> configuredQueues =
- new HashMap<FSQueueType, Set<String>>();
+ Map<FSQueueType, Set<String>> configuredQueues = new HashMap<>();
+
for (FSQueueType queueType : FSQueueType.values()) {
configuredQueues.put(queueType, new HashSet<String>());
}
@@ -368,10 +367,11 @@ public class AllocationFileLoaderService extends AbstractService {
parent = null;
}
loadQueue(parent, element, minQueueResources, maxQueueResources,
- queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights,
- queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts,
- fairSharePreemptionThresholds, queueAcls, reservationAcls,
- configuredQueues, reservableQueues, nonPreemptableQueues);
+ maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares,
+ queueWeights, queuePolicies, minSharePreemptionTimeouts,
+ fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls,
+ reservationAcls, configuredQueues, reservableQueues,
+ nonPreemptableQueues);
}
// Load placement policy and pass it configured queues
@@ -413,14 +413,15 @@ public class AllocationFileLoaderService extends AbstractService {
globalReservationQueueConfig.setReservationAgent(reservationAgent);
}
- AllocationConfiguration info = new AllocationConfiguration(minQueueResources,
- maxQueueResources, queueMaxApps, userMaxApps, queueWeights,
- queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault,
- queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies,
- defaultSchedPolicy, minSharePreemptionTimeouts,
- fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls,
- reservationAcls, newPlacementPolicy, configuredQueues,
- globalReservationQueueConfig, reservableQueues, nonPreemptableQueues);
+ AllocationConfiguration info =
+ new AllocationConfiguration(minQueueResources, maxQueueResources,
+ maxChildQueueResources, queueMaxApps, userMaxApps, queueWeights,
+ queueMaxAMShares, userMaxAppsDefault, queueMaxAppsDefault,
+ queueMaxResourcesDefault, queueMaxAMShareDefault, queuePolicies,
+ defaultSchedPolicy, minSharePreemptionTimeouts,
+ fairSharePreemptionTimeouts, fairSharePreemptionThresholds, queueAcls,
+ reservationAcls, newPlacementPolicy, configuredQueues,
+ globalReservationQueueConfig, reservableQueues, nonPreemptableQueues);
lastSuccessfulReload = clock.getTime();
lastReloadAttemptFailed = false;
@@ -433,8 +434,11 @@ public class AllocationFileLoaderService extends AbstractService {
*/
private void loadQueue(String parentName, Element element,
Map<String, Resource> minQueueResources,
- Map<String, Resource> maxQueueResources, Map<String, Integer> queueMaxApps,
- Map<String, Integer> userMaxApps, Map<String, Float> queueMaxAMShares,
+ Map<String, Resource> maxQueueResources,
+ Map<String, Resource> maxChildQueueResources,
+ Map<String, Integer> queueMaxApps,
+ Map<String, Integer> userMaxApps,
+ Map<String, Float> queueMaxAMShares,
Map<String, ResourceWeights> queueWeights,
Map<String, SchedulingPolicy> queuePolicies,
Map<String, Long> minSharePreemptionTimeouts,
@@ -463,8 +467,8 @@ public class AllocationFileLoaderService extends AbstractService {
if (parentName != null) {
queueName = parentName + "." + queueName;
}
- Map<QueueACL, AccessControlList> acls =
- new HashMap<QueueACL, AccessControlList>();
+
+ Map<QueueACL, AccessControlList> acls = new HashMap<>();
Map<ReservationACL, AccessControlList> racls = new HashMap<>();
NodeList fields = element.getChildNodes();
boolean isLeaf = true;
@@ -476,12 +480,19 @@ public class AllocationFileLoaderService extends AbstractService {
Element field = (Element) fieldNode;
if ("minResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
+ Resource val =
+ FairSchedulerConfiguration.parseResourceConfigValue(text);
minQueueResources.put(queueName, val);
} else if ("maxResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- Resource val = FairSchedulerConfiguration.parseResourceConfigValue(text);
+ Resource val =
+ FairSchedulerConfiguration.parseResourceConfigValue(text);
maxQueueResources.put(queueName, val);
+ } else if ("maxChildResources".equals(field.getTagName())) {
+ String text = ((Text)field.getFirstChild()).getData().trim();
+ Resource val =
+ FairSchedulerConfiguration.parseResourceConfigValue(text);
+ maxChildQueueResources.put(queueName, val);
} else if ("maxRunningApps".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
@@ -543,8 +554,8 @@ public class AllocationFileLoaderService extends AbstractService {
} else if ("queue".endsWith(field.getTagName()) ||
"pool".equals(field.getTagName())) {
loadQueue(queueName, field, minQueueResources, maxQueueResources,
- queueMaxApps, userMaxApps, queueMaxAMShares, queueWeights,
- queuePolicies, minSharePreemptionTimeouts,
+ maxChildQueueResources, queueMaxApps, userMaxApps, queueMaxAMShares,
+ queueWeights, queuePolicies, minSharePreemptionTimeouts,
fairSharePreemptionTimeouts, fairSharePreemptionThresholds,
queueAcls, resAcls, configuredQueues, reservableQueues,
nonPreemptableQueues);
@@ -574,9 +585,8 @@ public class AllocationFileLoaderService extends AbstractService {
&& !Resources.fitsIn(minQueueResources.get(queueName),
maxQueueResources.get(queueName))) {
LOG.warn(
- String.format(
- "Queue %s has max resources %s less than min resources %s",
- queueName, maxQueueResources.get(queueName),
+ String.format("Queue %s has max resources %s less than "
+ + "min resources %s", queueName, maxQueueResources.get(queueName),
minQueueResources.get(queueName)));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index ac384a1..73d56d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashSet;
@@ -1573,11 +1574,41 @@ public class FairScheduler extends
allocConf = queueInfo;
allocConf.getDefaultSchedulingPolicy().initialize(getClusterResource());
queueMgr.updateAllocationConfiguration(allocConf);
+ applyChildDefaults();
maxRunningEnforcer.updateRunnabilityOnReload();
}
}
}
+ /**
+ * After reloading the allocation config, the max resource settings for any
+ * ad hoc queues will be missing. This method goes through the queue manager's
+ * queue list and adds back the max resources settings for any ad hoc queues.
+ * Note that the new max resource settings will be based on the new config.
+ * The old settings are lost.
+ */
+ private void applyChildDefaults() {
+ Collection<FSQueue> queues = queueMgr.getQueues();
+ Set<String> configuredLeafQueues =
+ allocConf.getConfiguredQueues().get(FSQueueType.LEAF);
+ Set<String> configuredParentQueues =
+ allocConf.getConfiguredQueues().get(FSQueueType.PARENT);
+
+ for (FSQueue queue : queues) {
+ // If the queue is ad hoc and not root, apply the child defaults
+ if ((queue.getParent() != null) &&
+ !configuredLeafQueues.contains(queue.getName()) &&
+ !configuredParentQueues.contains(queue.getName())) {
+ Resource max =
+ allocConf.getMaxChildResources(queue.getParent().getName());
+
+ if (max != null) {
+ allocConf.setMaxResources(queue.getName(), max);
+ }
+ }
+ }
+ }
+
@Override
public List<ApplicationAttemptId> getAppsInQueue(String queueName) {
FSQueue queue = queueMgr.getQueue(queueName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 8d308dc..964c72b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -39,6 +39,9 @@ import org.xml.sax.SAXException;
import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
+import java.util.Iterator;
+import java.util.Set;
+import org.apache.hadoop.yarn.api.records.Resource;
/**
* Maintains a list of queues as well as scheduling parameters for each queue,
* such as guaranteed share allocations, from the fair scheduler config file.
@@ -173,15 +176,42 @@ public class QueueManager {
}
/**
- * Creates a leaf or parent queue based on what is specified in 'queueType'
- * and places it in the tree. Creates any parents that don't already exist.
+ * Create a leaf or parent queue based on what is specified in
+ * {@code queueType} and place it in the tree. Create any parents that don't
+ * already exist.
*
- * @return
- * the created queue, if successful. null if not allowed (one of the parent
- * queues in the queue name is already a leaf queue)
+ * @return the created queue, if successful or null if not allowed (one of the
+ * parent queues in the queue name is already a leaf queue)
*/
- private FSQueue createQueue(String name, FSQueueType queueType) {
- List<String> newQueueNames = new ArrayList<String>();
+ @VisibleForTesting
+ FSQueue createQueue(String name, FSQueueType queueType) {
+ List<String> newQueueNames = new ArrayList<>();
+ FSParentQueue parent = buildNewQueueList(name, newQueueNames);
+ FSQueue queue = null;
+
+ if (parent != null) {
+ // Now that we know everything worked out, make all the queues
+ // and add them to the map.
+ queue = createNewQueues(queueType, parent, newQueueNames);
+ }
+
+ return queue;
+ }
+
+ /**
+ * Compile a list of all parent queues of the given queue name that do not
+ * already exist. The queue names will be added to the {@code newQueueNames}
+ * list. The list will be in order of increasing queue depth. The first
+ * element of the list will be the parent closest to the root. The last
+ * element added will be the queue to be created. This method returns the
+ * deepest parent that does exist.
+ *
+ * @param name the fully qualified name of the queue to create
+ * @param newQueueNames the list to which to add non-existent queues
+ * @return the deepest existing parent queue
+ */
+ private FSParentQueue buildNewQueueList(String name,
+ List<String> newQueueNames) {
newQueueNames.add(name);
int sepIndex = name.length();
FSParentQueue parent = null;
@@ -195,60 +225,118 @@ public class QueueManager {
throw new InvalidQueueNameException("Illegal node name at offset " +
(sepIndex+1) + " for queue name " + name);
}
- FSQueue queue;
- String curName = null;
- curName = name.substring(0, sepIndex);
- queue = queues.get(curName);
+
+ String curName = name.substring(0, sepIndex);
+ FSQueue queue = queues.get(curName);
if (queue == null) {
- newQueueNames.add(curName);
+ newQueueNames.add(0, curName);
} else {
if (queue instanceof FSParentQueue) {
parent = (FSParentQueue)queue;
- break;
- } else {
- return null;
}
+
+ // If the queue isn't a parent queue, parent will still be null when
+ // we break
+
+ break;
}
}
-
- // At this point, parent refers to the deepest existing parent of the
- // queue to create.
- // Now that we know everything worked out, make all the queues
- // and add them to the map.
+
+ return parent;
+ }
+
+ /**
+ * Create all queues in the {@code newQueueNames} list. The list must be in
+ * order of increasing depth. All but the last element in the list will be
+ * created as parent queues. The last element will be created as the type
+ * specified by the {@code queueType} parameter. The first queue will be
+ * created as a child of the {@code topParent} queue. All subsequent queues
+ * will be created as a child of the previously created queue.
+ *
+ * @param queueType the type of the last queue to create
+ * @param topParent the parent of the first queue to create
+ * @param newQueueNames the list of queues to create
+ * @return the last queue created
+ */
+ private FSQueue createNewQueues(FSQueueType queueType,
+ FSParentQueue topParent, List<String> newQueueNames) {
AllocationConfiguration queueConf = scheduler.getAllocationConfiguration();
- FSLeafQueue leafQueue = null;
- for (int i = newQueueNames.size()-1; i >= 0; i--) {
- String queueName = newQueueNames.get(i);
- if (i == 0 && queueType != FSQueueType.PARENT) {
- leafQueue = new FSLeafQueue(name, scheduler, parent);
+ Iterator<String> i = newQueueNames.iterator();
+ FSParentQueue parent = topParent;
+ FSQueue queue = null;
+
+ while (i.hasNext()) {
+ FSParentQueue newParent = null;
+ String queueName = i.next();
+
+ // Only create a leaf queue at the very end
+ if (!i.hasNext() && (queueType != FSQueueType.PARENT)) {
+ FSLeafQueue leafQueue = new FSLeafQueue(queueName, scheduler, parent);
+
try {
leafQueue.setPolicy(queueConf.getDefaultSchedulingPolicy());
} catch (AllocationConfigurationException ex) {
LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy() + " on new leaf queue.", ex);
+ + queueConf.getDefaultSchedulingPolicy()
+ + " on new leaf queue.", ex);
}
- parent.addChildQueue(leafQueue);
- queues.put(leafQueue.getName(), leafQueue);
+
leafQueues.add(leafQueue);
- leafQueue.updatePreemptionVariables();
- return leafQueue;
+ queue = leafQueue;
} else {
- FSParentQueue newParent = new FSParentQueue(queueName, scheduler, parent);
+ newParent = new FSParentQueue(queueName, scheduler, parent);
+
try {
newParent.setPolicy(queueConf.getDefaultSchedulingPolicy());
} catch (AllocationConfigurationException ex) {
LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy() + " on new parent queue.", ex);
+ + queueConf.getDefaultSchedulingPolicy()
+ + " on new parent queue.", ex);
}
- parent.addChildQueue(newParent);
- queues.put(newParent.getName(), newParent);
- newParent.updatePreemptionVariables();
- parent = newParent;
+
+ queue = newParent;
}
+
+ parent.addChildQueue(queue);
+ setChildResourceLimits(parent, queue, queueConf);
+ queues.put(queue.getName(), queue);
+ queue.updatePreemptionVariables();
+
+ // If we just created a leaf node, the newParent is null, but that's OK
+ // because we only create a leaf node in the very last iteration.
+ parent = newParent;
}
- return parent;
+ return queue;
+ }
+
+ /**
+ * For the given child queue, set the max resources based on the
+ * parent queue's default child resource settings. This method assumes that
+ * the child queue is ad hoc and hence does not do any safety checks around
+ * overwriting existing max resource settings.
+ *
+ * @param parent the parent queue
+ * @param child the child queue
+ * @param queueConf the {@link AllocationConfiguration}
+ */
+ void setChildResourceLimits(FSParentQueue parent, FSQueue child,
+ AllocationConfiguration queueConf) {
+ Map<FSQueueType, Set<String>> configuredQueues =
+ queueConf.getConfiguredQueues();
+
+ // Ad hoc queues do not exist in the configured queues map
+ if (!configuredQueues.get(FSQueueType.LEAF).contains(child.getName()) &&
+ !configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) {
+ // For ad hoc queues, set their max reource allocations based on
+ // their parents' default child settings.
+ Resource maxChild = queueConf.getMaxChildResources(parent.getName());
+
+ if (maxChild != null) {
+ queueConf.setMaxResources(child.getName(), maxChild);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 11d4981..12c3fa9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -179,12 +179,16 @@ public class TestAllocationFileLoaderService {
out.println("<queue name=\"queueE\">");
out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
out.println("</queue>");
- //Make queue F a parent queue without configured leaf queues using the 'type' attribute
+ // Make queue F a parent queue without configured leaf queues using the
+ // 'type' attribute
out.println("<queue name=\"queueF\" type=\"parent\" >");
+ out.println("<maxChildResources>2048mb,64vcores</maxChildResources>");
out.println("</queue>");
// Create hierarchical queues G,H, with different min/fair share preemption
- // timeouts and preemption thresholds
+ // timeouts and preemption thresholds. Also add a child default to make sure
+ // it doesn't impact queue H.
out.println("<queue name=\"queueG\">");
+ out.println("<maxChildResources>2048mb,64vcores</maxChildResources>");
out.println("<fairSharePreemptionTimeout>120</fairSharePreemptionTimeout>");
out.println("<minSharePreemptionTimeout>50</minSharePreemptionTimeout>");
out.println("<fairSharePreemptionThreshold>0.6</fairSharePreemptionThreshold>");
@@ -240,6 +244,12 @@ public class TestAllocationFileLoaderService {
queueConf.getMaxResources("root.queueD"));
assertEquals(Resources.createResource(4096, 100),
queueConf.getMaxResources("root.queueE"));
+ assertEquals(Resources.createResource(4096, 100),
+ queueConf.getMaxResources("root.queueF"));
+ assertEquals(Resources.createResource(4096, 100),
+ queueConf.getMaxResources("root.queueG"));
+ assertEquals(Resources.createResource(4096, 100),
+ queueConf.getMaxResources("root.queueG.queueH"));
assertEquals(Resources.createResource(1024, 0),
queueConf.getMinResources("root.queueA"));
@@ -251,8 +261,33 @@ public class TestAllocationFileLoaderService {
queueConf.getMinResources("root.queueD"));
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root.queueE"));
-
- assertEquals(15, queueConf.getQueueMaxApps("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
+ assertEquals(Resources.createResource(0),
+ queueConf.getMinResources("root.queueF"));
+ assertEquals(Resources.createResource(0),
+ queueConf.getMinResources("root.queueG"));
+ assertEquals(Resources.createResource(0),
+ queueConf.getMinResources("root.queueG.queueH"));
+
+ assertNull("Max child resources unexpectedly set for queue root.queueA",
+ queueConf.getMaxChildResources("root.queueA"));
+ assertNull("Max child resources unexpectedly set for queue root.queueB",
+ queueConf.getMaxChildResources("root.queueB"));
+ assertNull("Max child resources unexpectedly set for queue root.queueC",
+ queueConf.getMaxChildResources("root.queueC"));
+ assertNull("Max child resources unexpectedly set for queue root.queueD",
+ queueConf.getMaxChildResources("root.queueD"));
+ assertNull("Max child resources unexpectedly set for queue root.queueE",
+ queueConf.getMaxChildResources("root.queueE"));
+ assertEquals(Resources.createResource(2048, 64),
+ queueConf.getMaxChildResources("root.queueF"));
+ assertEquals(Resources.createResource(2048, 64),
+ queueConf.getMaxChildResources("root.queueG"));
+ assertNull("Max child resources unexpectedly set for "
+ + "queue root.queueG.queueH",
+ queueConf.getMaxChildResources("root.queueG.queueH"));
+
+ assertEquals(15, queueConf.getQueueMaxApps("root."
+ + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(15, queueConf.getQueueMaxApps("root.queueA"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueB"));
assertEquals(15, queueConf.getQueueMaxApps("root.queueC"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 87670c4..da9d667 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -139,8 +139,6 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf = createConfiguration();
resourceManager = new MockRM(conf);
- // TODO: This test should really be using MockRM. For now starting stuff
- // that is needed at a bare minimum.
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
resourceManager.getRMContext().getStateStore().start();
@@ -335,8 +333,14 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test fair shares when max resources are set but are too high to impact
+ * the shares.
+ *
+ * @throws IOException if scheduler reinitialization fails
+ */
@Test
- public void testFairShareWithMaxResources() throws IOException {
+ public void testFairShareWithHighMaxResources() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB maxResources,
// the sum of queueA and queueB maxResources is more than
@@ -376,11 +380,184 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(
"queueA", false);
// queueA's weight is 0.25, so its fair share should be 2 * 1024.
- assertEquals(2 * 1024, queue.getFairShare().getMemorySize());
+ assertEquals("Queue A did not get its expected fair share",
+ 2 * 1024, queue.getFairShare().getMemorySize());
// queueB's weight is 0.75, so its fair share should be 6 * 1024.
queue = scheduler.getQueueManager().getLeafQueue(
"queueB", false);
- assertEquals(6 * 1024, queue.getFairShare().getMemorySize());
+ assertEquals("Queue B did not get its expected fair share",
+ 6 * 1024, queue.getFairShare().getMemorySize());
+ }
+
+ /**
+ * Test fair shares when max resources are set and are low enough to impact
+ * the shares.
+ *
+ * @throws IOException if scheduler reinitialization fails
+ */
+ @Test
+ public void testFairShareWithLowMaxResources() throws IOException {
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\">");
+ out.println(" <maxResources>1024 mb 1 vcores</maxResources>");
+ out.println(" <weight>0.75</weight>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"queueB\">");
+ out.println(" <maxResources>3072 mb 3 vcores</maxResources>");
+ out.println(" <weight>0.25</weight>");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest(1024, 1, "queueA", "user1", 2);
+ ApplicationAttemptId attId2 =
+ createSchedulingRequest(1024, 1, "queueB", "user1", 4);
+
+ scheduler.update();
+
+ FSLeafQueue queue =
+ scheduler.getQueueManager().getLeafQueue("queueA", false);
+ // queueA's weight is 0.5, so its fair share should be 6GB, but it's
+ // capped at 1GB.
+ assertEquals("Queue A did not get its expected fair share",
+ 1 * 1024, queue.getFairShare().getMemorySize());
+ // queueB's weight is 0.5, so its fair share should be 2GB, but the
+ // other queue is capped at 1GB, so queueB's share is 7GB,
+ // capped at 3GB.
+ queue = scheduler.getQueueManager().getLeafQueue(
+ "queueB", false);
+ assertEquals("Queue B did not get its expected fair share",
+ 3 * 1024, queue.getFairShare().getMemorySize());
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+
+ // App 1 should be running with 1 container
+ assertEquals("App 1 is not running with the correct number of containers",
+ 1, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
+ // App 2 should be running with 3 containers
+ assertEquals("App 2 is not running with the correct number of containers",
+ 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
+ }
+
+ /**
+ * Test the child max resource settings.
+ *
+ * @throws IOException if scheduler reinitialization fails
+ */
+ @Test
+ public void testChildMaxResources() throws IOException {
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\" type=\"parent\">");
+ out.println(" <maxChildResources>2048mb,2vcores</maxChildResources>");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ ApplicationAttemptId attId1 =
+ createSchedulingRequest(1024, 1, "queueA.queueB", "user1", 8);
+ ApplicationAttemptId attId2 =
+ createSchedulingRequest(1024, 1, "queueA.queueC", "user1", 8);
+
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent nodeEvent = new NodeUpdateSchedulerEvent(node1);
+
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+
+ // Apps should be running with 2 containers
+ assertEquals("App 1 is not running with the correct number of containers",
+ 2, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
+ assertEquals("App 2 is not running with the correct number of containers",
+ 2, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
+
+ out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\" type=\"parent\">");
+ out.println(" <maxChildResources>3072mb,3vcores</maxChildResources>");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ scheduler.update();
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(nodeEvent);
+
+ // Apps should be running with 3 containers now
+ assertEquals("App 1 is not running with the correct number of containers",
+ 3, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
+ assertEquals("App 2 is not running with the correct number of containers",
+ 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
+
+ out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println(" <queue name=\"queueA\" type=\"parent\">");
+ out.println(" <maxChildResources>1024mb,1vcores</maxChildResources>");
+ out.println(" </queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ scheduler.update();
+ scheduler.handle(nodeEvent);
+
+ // Apps still should be running with 3 containers because we don't preempt
+ assertEquals("App 1 is not running with the correct number of containers",
+ 3, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
+ assertEquals("App 2 is not running with the correct number of containers",
+ 3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 33d4419..a9c3788 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Before;
import org.junit.Test;
@@ -33,27 +34,41 @@ public class TestQueueManager {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
private Set<FSQueue> notEmptyQueues;
+ private FairScheduler scheduler;
@Before
public void setUp() throws Exception {
conf = new FairSchedulerConfiguration();
- FairScheduler scheduler = mock(FairScheduler.class);
+ scheduler = mock(FairScheduler.class);
+
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
+
+ // Set up some queues to test default child max resource inheritance
+ allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test");
+ allocConf.setMaxChildResources("root.test",
+ Resources.createResource(8192, 256));
+ allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA");
+ allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB");
+
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
when(scheduler.getConf()).thenReturn(conf);
+
SystemClock clock = SystemClock.getInstance();
+
when(scheduler.getClock()).thenReturn(clock);
- notEmptyQueues = new HashSet<FSQueue>();
+ notEmptyQueues = new HashSet<>();
queueManager = new QueueManager(scheduler) {
@Override
public boolean isEmpty(FSQueue queue) {
return !notEmptyQueues.contains(queue);
}
};
+
FSQueueMetrics.forQueue("root", null, true, conf);
+
queueManager.initialize(conf);
}
-
+
@Test
public void testReloadTurnsLeafQueueIntoParent() throws Exception {
updateConfiguredLeafQueues(queueManager, "queue1");
@@ -143,4 +158,149 @@ public class TestQueueManager {
allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
+
+ /**
+ * Test simple leaf queue creation.
+ */
+ @Test
+ public void testCreateLeafQueue() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
+
+ assertNotNull("Leaf queue root.queue1 was not created",
+ queueManager.getLeafQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+ }
+
+ /**
+ * Test creation of a leaf queue and its parent.
+ */
+ @Test
+ public void testCreateLeafQueueAndParent() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ }
+
+ /**
+ * Test creation of leaf and parent child queues when the parent queue has
+ * child defaults set. In this test we rely on the root.test,
+ * root.test.childA and root.test.childB queues that are created in the
+ * {@link #setUp()} method.
+ */
+ @Test
+ public void testCreateQueueWithChildDefaults() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
+
+ assertNotNull("Leaf queue root.test.childC was not created",
+ queueManager.getLeafQueue("root.test.childC", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.test.childC", q1.getName());
+ assertEquals("Max resources for root.queue1 were not inherited from "
+ + "parent's max child resources", Resources.createResource(8192, 256),
+ allocConf.getMaxResources("root.test.childC"));
+
+ FSQueue q2 = queueManager.createQueue("root.test.childD",
+ FSQueueType.PARENT);
+
+ assertNotNull("Leaf queue root.test.childD was not created",
+ queueManager.getParentQueue("root.test.childD", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.test.childD", q2.getName());
+ assertEquals("Max resources for root.test.childD were not inherited "
+ + "from parent's max child resources",
+ Resources.createResource(8192, 256),
+ allocConf.getMaxResources("root.test.childD"));
+
+ // Check that the childA and childB queues weren't impacted
+ // by the child defaults
+ assertNotNull("Leaf queue root.test.childA was not created during setup",
+ queueManager.getLeafQueue("root.test.childA", false));
+ assertEquals("Max resources for root.test.childA were inherited from "
+ + "parent's max child resources", Resources.unbounded(),
+ allocConf.getMaxResources("root.test.childA"));
+ assertNotNull("Leaf queue root.test.childB was not created during setup",
+ queueManager.getParentQueue("root.test.childB", false));
+ assertEquals("Max resources for root.test.childB were inherited from "
+ + "parent's max child resources", Resources.unbounded(),
+ allocConf.getMaxResources("root.test.childB"));
+ }
+
+ /**
+ * Test creation of a leaf queue with no resource limits.
+ */
+ @Test
+ public void testCreateLeafQueueWithDefaults() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+ FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
+
+ assertNotNull("Leaf queue root.queue1 was not created",
+ queueManager.getLeafQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+
+ // Min default is 0,0
+ assertEquals("Min resources were not set to default",
+ Resources.createResource(0, 0),
+ allocConf.getMinResources("root.queue1"));
+
+ // Max default is unbounded
+ assertEquals("Max resources were not set to default", Resources.unbounded(),
+ allocConf.getMaxResources("root.queue1"));
+ }
+
+ /**
+ * Test creation of a simple parent queue.
+ */
+ @Test
+ public void testCreateParentQueue() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.PARENT);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+ }
+
+ /**
+ * Test creation of a parent queue and its parent.
+ */
+ @Test
+ public void testCreateParentQueueAndParent() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.PARENT);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getParentQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20f0eb87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 977d729..ecbb309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -99,6 +99,8 @@ The allocation file must be in XML format. The format contains five types of ele
* maxResources: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit.
+ * maxChildResources: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly.
+
* maxRunningApps: limit the number of apps from the queue to run at once
* maxAMShare: limit the fraction of the queue's fair share that can be used to run application masters. This property can only be used for leaf queues. For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% of both the memory and CPU fair share. The value of -1.0f will disable this feature and the amShare will not be checked. The default value is 0.5f.
@@ -176,6 +178,7 @@ The allocation file must be in XML format. The format contains five types of ele
user queues under it -->
<queue name="secondary_group_queue" type="parent">
<weight>3.0</weight>
+ <maxChildResources>4096 mb,4vcores</maxChildResources>
</queue>
<user name="sample_user">
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDFS-10567. Improve plan command
help message. Contributed by Xiaobing Zhou.
Posted by cn...@apache.org.
HDFS-10567. Improve plan command help message. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02abd131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02abd131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02abd131
Branch: refs/heads/HADOOP-13345
Commit: 02abd131b857a89d9fc21507296603120bb50810
Parents: 9daa997
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Aug 15 19:54:06 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Aug 15 19:58:57 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/tools/DiskBalancer.java | 29 ++++++++++++--------
1 file changed, 18 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02abd131/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
index 70912d0..1ed2fdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
@@ -266,33 +266,40 @@ public class DiskBalancer extends Configured implements Tool {
private void addPlanCommands(Options opt) {
Option plan = OptionBuilder.withLongOpt(PLAN)
- .withDescription("creates a plan for datanode.")
+ .withDescription("Hostname, IP address or UUID of datanode " +
+ "for which a plan is created.")
.hasArg()
.create();
getPlanOptions().addOption(plan);
opt.addOption(plan);
- Option outFile = OptionBuilder.withLongOpt(OUTFILE)
- .hasArg()
- .withDescription("File to write output to, if not specified " +
- "defaults will be used.")
+ Option outFile = OptionBuilder.withLongOpt(OUTFILE).hasArg()
+ .withDescription(
+ "Local path of file to write output to, if not specified "
+ + "defaults will be used.")
.create();
getPlanOptions().addOption(outFile);
opt.addOption(outFile);
- Option bandwidth = OptionBuilder.withLongOpt(BANDWIDTH)
- .hasArg()
- .withDescription("Maximum disk bandwidth to be consumed by " +
- "diskBalancer. e.g. 10")
+ Option bandwidth = OptionBuilder.withLongOpt(BANDWIDTH).hasArg()
+ .withDescription(
+ "Maximum disk bandwidth (MB/s) in integer to be consumed by "
+ + "diskBalancer. e.g. 10 MB/s.")
.create();
getPlanOptions().addOption(bandwidth);
opt.addOption(bandwidth);
Option threshold = OptionBuilder.withLongOpt(THRESHOLD)
.hasArg()
- .withDescription("Percentage skew that we" +
- "tolerate before diskbalancer starts working e.g. 10")
+ .withDescription("Percentage of data skew that is tolerated before"
+ + " disk balancer starts working. For example, if"
+ + " total data on a 2 disk node is 100 GB then disk"
+ + " balancer calculates the expected value on each disk,"
+ + " which is 50 GB. If the tolerance is 10% then data"
+ + " on a single disk needs to be more than 60 GB"
+ + " (50 GB + 10% tolerance value) for Disk balancer to"
+ + " balance the disks.")
.create();
getPlanOptions().addOption(threshold);
opt.addOption(threshold);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-13190. Mention
LoadBalancingKMSClientProvider in KMS HA documentation. Contributed by
Wei-Chiu Chuang.
Posted by cn...@apache.org.
HADOOP-13190. Mention LoadBalancingKMSClientProvider in KMS HA documentation. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db719ef1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db719ef1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db719ef1
Branch: refs/heads/HADOOP-13345
Commit: db719ef125b11b01eab3353e2dc4b48992bf88d5
Parents: d892ae9
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Aug 11 12:27:09 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Aug 11 12:27:09 2016 -0700
----------------------------------------------------------------------
.../hadoop-kms/src/site/markdown/index.md.vm | 66 +++++++++++++++++---
1 file changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db719ef1/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index d50b0f4..889dbaf1 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -19,6 +19,8 @@
Hadoop Key Management Server (KMS) - Documentation Sets
=======================================================
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
Hadoop KMS is a cryptographic key management server based on Hadoop's **KeyProvider** API.
It provides a client and a server components which communicate over HTTP using a REST API.
@@ -34,6 +36,18 @@ KMS Client Configuration
The KMS client `KeyProvider` uses the **kms** scheme, and the embedded URL must be the URL of the KMS. For example, for a KMS running on `http://localhost:9600/kms`, the KeyProvider URI is `kms://http@localhost:9600/kms`. And, for a KMS running on `https://localhost:9600/kms`, the KeyProvider URI is `kms://https@localhost:9600/kms`
+The following is an example to configure HDFS NameNode as a KMS client in
+`hdfs-site.xml`:
+
+ <property>
+ <name>dfs.encryption.key.provider.uri</name>
+ <value>kms://http@localhost:9600/kms</value>
+ <description>
+ The KeyProvider to use when interacting with encryption keys used
+ when reading and writing to an encryption zone.
+ </description>
+ </property>
+
KMS
---
@@ -623,13 +637,51 @@ Additionally, KMS delegation token secret manager can be configured with the fol
</property>
```
-$H3 Using Multiple Instances of KMS Behind a Load-Balancer or VIP
-
-KMS supports multiple KMS instances behind a load-balancer or VIP for scalability and for HA purposes.
-
-When using multiple KMS instances behind a load-balancer or VIP, requests from the same user may be handled by different KMS instances.
-
-KMS instances behind a load-balancer or VIP must be specially configured to work properly as a single logical service.
+$H3 High Availability
+
+Multiple KMS instances may be used to provide high availability and scalability.
+Currently there are two approaches to supporting multiple KMS instances:
+running KMS instances behind a load-balancer/VIP,
+or using LoadBalancingKMSClientProvider.
+
+In both approaches, KMS instances must be specially configured to work properly
+as a single logical service, because requests from the same client may be
+handled by different KMS instances. In particular,
+Kerberos Principals Configuration, HTTP Authentication Signature and Delegation
+Tokens require special attention.
+
+$H4 Behind a Load-Balancer or VIP
+
+Because KMS clients and servers communicate via a REST API over HTTP,
+Load-balancer or VIP may be used to distribute incoming traffic to achieve
+scalability and HA. In this mode, clients are unaware of multiple KMS instances
+at the server-side.
+
+$H4 Using LoadBalancingKMSClientProvider
+
+An alternative to running multiple KMS instances behind a load-balancer or VIP,
+is to use LoadBalancingKMSClientProvider. Using this approach, a KMS client
+(for example, a HDFS NameNode) is aware of multiple KMS instances, and it sends
+requests to them in a round-robin fashion. LoadBalancingKMSClientProvider is
+implicitly used when more than one URI is specified in
+`dfs.encryption.key.provider.uri`.
+
+The following example in `hdfs-site.xml` configures two KMS
+instances, `kms01.example.com` and `kms02.example.com`.
+The hostnames are separated by semi-colons, and all KMS instances must run
+on the same port.
+
+ <property>
+ <name>dfs.encryption.key.provider.uri</name>
+ <value>kms://https@kms01.example.com;kms02.example.com:9600/kms</value>
+ <description>
+ The KeyProvider to use when interacting with encryption keys used
+ when reading and writing to an encryption zone.
+ </description>
+ </property>
+
+If a request to a KMS instance fails, clients retry with the next instance. The
+request is returned as failure only if all instances fail.
$H4 HTTP Kerberos Principals Configuration
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: MAPREDUCE-6690. Limit the number
of resources a single map reduce job can submit for localization. Contributed
by Chris Trezzo
Posted by cn...@apache.org.
MAPREDUCE-6690. Limit the number of resources a single map reduce job can submit for localization. Contributed by Chris Trezzo
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f80a7298
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f80a7298
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f80a7298
Branch: refs/heads/HADOOP-13345
Commit: f80a7298325a4626638ee24467e2012442e480d4
Parents: 7f05ff7
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 17 16:21:20 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Aug 17 16:22:31 2016 +0000
----------------------------------------------------------------------
.../hadoop/mapreduce/JobResourceUploader.java | 214 +++++++++--
.../apache/hadoop/mapreduce/MRJobConfig.java | 28 ++
.../ClientDistributedCacheManager.java | 15 +-
.../src/main/resources/mapred-default.xml | 30 ++
.../mapreduce/TestJobResourceUploader.java | 355 +++++++++++++++++++
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 166 ++++++++-
6 files changed, 776 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index fa4dd86..15dbc13 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -21,12 +21,16 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -34,6 +38,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
+import com.google.common.annotations.VisibleForTesting;
+
@InterfaceAudience.Private
@InterfaceStability.Unstable
class JobResourceUploader {
@@ -86,31 +92,37 @@ class JobResourceUploader {
FsPermission mapredSysPerms =
new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
FileSystem.mkdirs(jtFs, submitJobDir, mapredSysPerms);
- // add all the command line files/ jars and archive
- // first copy them to jobtrackers filesystem
- uploadFiles(conf, submitJobDir, mapredSysPerms, replication);
- uploadLibJars(conf, submitJobDir, mapredSysPerms, replication);
- uploadArchives(conf, submitJobDir, mapredSysPerms, replication);
- uploadJobJar(job, submitJobDir, replication);
+ Collection<String> files = conf.getStringCollection("tmpfiles");
+ Collection<String> libjars = conf.getStringCollection("tmpjars");
+ Collection<String> archives = conf.getStringCollection("tmparchives");
+ String jobJar = job.getJar();
+
+ Map<URI, FileStatus> statCache = new HashMap<URI, FileStatus>();
+ checkLocalizationLimits(conf, files, libjars, archives, jobJar, statCache);
+
+ uploadFiles(conf, files, submitJobDir, mapredSysPerms, replication);
+ uploadLibJars(conf, libjars, submitJobDir, mapredSysPerms, replication);
+ uploadArchives(conf, archives, submitJobDir, mapredSysPerms, replication);
+ uploadJobJar(job, jobJar, submitJobDir, replication);
addLog4jToDistributedCache(job, submitJobDir);
// set the timestamps of the archives and files
// set the public/private visibility of the archives and files
- ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf);
+ ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(conf,
+ statCache);
// get DelegationToken for cached file
ClientDistributedCacheManager.getDelegationTokens(conf,
job.getCredentials());
}
- private void uploadFiles(Configuration conf, Path submitJobDir,
- FsPermission mapredSysPerms, short submitReplication) throws IOException {
- String files = conf.get("tmpfiles");
+ private void uploadFiles(Configuration conf, Collection<String> files,
+ Path submitJobDir, FsPermission mapredSysPerms, short submitReplication)
+ throws IOException {
Path filesDir = JobSubmissionFiles.getJobDistCacheFiles(submitJobDir);
- if (files != null) {
+ if (!files.isEmpty()) {
FileSystem.mkdirs(jtFs, filesDir, mapredSysPerms);
- String[] fileArr = files.split(",");
- for (String tmpFile : fileArr) {
+ for (String tmpFile : files) {
URI tmpURI = null;
try {
tmpURI = new URI(tmpFile);
@@ -130,14 +142,13 @@ class JobResourceUploader {
}
}
- private void uploadLibJars(Configuration conf, Path submitJobDir,
- FsPermission mapredSysPerms, short submitReplication) throws IOException {
- String libjars = conf.get("tmpjars");
+ private void uploadLibJars(Configuration conf, Collection<String> libjars,
+ Path submitJobDir, FsPermission mapredSysPerms, short submitReplication)
+ throws IOException {
Path libjarsDir = JobSubmissionFiles.getJobDistCacheLibjars(submitJobDir);
- if (libjars != null) {
+ if (!libjars.isEmpty()) {
FileSystem.mkdirs(jtFs, libjarsDir, mapredSysPerms);
- String[] libjarsArr = libjars.split(",");
- for (String tmpjars : libjarsArr) {
+ for (String tmpjars : libjars) {
Path tmp = new Path(tmpjars);
Path newPath =
copyRemoteFiles(libjarsDir, tmp, conf, submitReplication);
@@ -157,14 +168,13 @@ class JobResourceUploader {
}
}
- private void uploadArchives(Configuration conf, Path submitJobDir,
- FsPermission mapredSysPerms, short submitReplication) throws IOException {
- String archives = conf.get("tmparchives");
+ private void uploadArchives(Configuration conf, Collection<String> archives,
+ Path submitJobDir, FsPermission mapredSysPerms, short submitReplication)
+ throws IOException {
Path archivesDir = JobSubmissionFiles.getJobDistCacheArchives(submitJobDir);
- if (archives != null) {
+ if (!archives.isEmpty()) {
FileSystem.mkdirs(jtFs, archivesDir, mapredSysPerms);
- String[] archivesArr = archives.split(",");
- for (String tmpArchives : archivesArr) {
+ for (String tmpArchives : archives) {
URI tmpURI;
try {
tmpURI = new URI(tmpArchives);
@@ -185,9 +195,8 @@ class JobResourceUploader {
}
}
- private void uploadJobJar(Job job, Path submitJobDir, short submitReplication)
- throws IOException {
- String jobJar = job.getJar();
+ private void uploadJobJar(Job job, String jobJar, Path submitJobDir,
+ short submitReplication) throws IOException {
if (jobJar != null) { // copy jar to JobTracker's fs
// use jar name if job is not named.
if ("".equals(job.getJobName())) {
@@ -208,6 +217,155 @@ class JobResourceUploader {
}
}
+ /**
+ * Verify that the resources this job is going to localize are within the
+ * localization limits.
+ */
+ @VisibleForTesting
+ void checkLocalizationLimits(Configuration conf, Collection<String> files,
+ Collection<String> libjars, Collection<String> archives, String jobJar,
+ Map<URI, FileStatus> statCache) throws IOException {
+
+ LimitChecker limitChecker = new LimitChecker(conf);
+ if (!limitChecker.hasLimits()) {
+ // there are no limits set, so we are done.
+ return;
+ }
+
+ // Get the files and archives that are already in the distributed cache
+ Collection<String> dcFiles =
+ conf.getStringCollection(MRJobConfig.CACHE_FILES);
+ Collection<String> dcArchives =
+ conf.getStringCollection(MRJobConfig.CACHE_ARCHIVES);
+
+ for (String path : dcFiles) {
+ explorePath(conf, new Path(path), limitChecker, statCache);
+ }
+
+ for (String path : dcArchives) {
+ explorePath(conf, new Path(path), limitChecker, statCache);
+ }
+
+ for (String path : files) {
+ explorePath(conf, new Path(path), limitChecker, statCache);
+ }
+
+ for (String path : libjars) {
+ explorePath(conf, new Path(path), limitChecker, statCache);
+ }
+
+ for (String path : archives) {
+ explorePath(conf, new Path(path), limitChecker, statCache);
+ }
+
+ if (jobJar != null) {
+ explorePath(conf, new Path(jobJar), limitChecker, statCache);
+ }
+ }
+
+ @VisibleForTesting
+ protected static final String MAX_RESOURCE_ERR_MSG =
+ "This job has exceeded the maximum number of submitted resources";
+ @VisibleForTesting
+ protected static final String MAX_TOTAL_RESOURCE_MB_ERR_MSG =
+ "This job has exceeded the maximum size of submitted resources";
+ @VisibleForTesting
+ protected static final String MAX_SINGLE_RESOURCE_MB_ERR_MSG =
+ "This job has exceeded the maximum size of a single submitted resource";
+
+ private static class LimitChecker {
+ LimitChecker(Configuration conf) {
+ this.maxNumOfResources =
+ conf.getInt(MRJobConfig.MAX_RESOURCES,
+ MRJobConfig.MAX_RESOURCES_DEFAULT);
+ this.maxSizeMB =
+ conf.getLong(MRJobConfig.MAX_RESOURCES_MB,
+ MRJobConfig.MAX_RESOURCES_MB_DEFAULT);
+ this.maxSizeOfResourceMB =
+ conf.getLong(MRJobConfig.MAX_SINGLE_RESOURCE_MB,
+ MRJobConfig.MAX_SINGLE_RESOURCE_MB_DEFAULT);
+ this.totalConfigSizeBytes = maxSizeMB * 1024 * 1024;
+ this.totalConfigSizeOfResourceBytes = maxSizeOfResourceMB * 1024 * 1024;
+ }
+
+ private long totalSizeBytes = 0;
+ private int totalNumberOfResources = 0;
+ private long currentMaxSizeOfFileBytes = 0;
+ private final long maxSizeMB;
+ private final int maxNumOfResources;
+ private final long maxSizeOfResourceMB;
+ private final long totalConfigSizeBytes;
+ private final long totalConfigSizeOfResourceBytes;
+
+ private boolean hasLimits() {
+ return maxNumOfResources > 0 || maxSizeMB > 0 || maxSizeOfResourceMB > 0;
+ }
+
+ private void addFile(Path p, long fileSizeBytes) throws IOException {
+ totalNumberOfResources++;
+ totalSizeBytes += fileSizeBytes;
+ if (fileSizeBytes > currentMaxSizeOfFileBytes) {
+ currentMaxSizeOfFileBytes = fileSizeBytes;
+ }
+
+ if (totalConfigSizeBytes > 0 && totalSizeBytes > totalConfigSizeBytes) {
+ throw new IOException(MAX_TOTAL_RESOURCE_MB_ERR_MSG + " (Max: "
+ + maxSizeMB + "MB).");
+ }
+
+ if (maxNumOfResources > 0 &&
+ totalNumberOfResources > maxNumOfResources) {
+ throw new IOException(MAX_RESOURCE_ERR_MSG + " (Max: "
+ + maxNumOfResources + ").");
+ }
+
+ if (totalConfigSizeOfResourceBytes > 0
+ && currentMaxSizeOfFileBytes > totalConfigSizeOfResourceBytes) {
+ throw new IOException(MAX_SINGLE_RESOURCE_MB_ERR_MSG + " (Max: "
+ + maxSizeOfResourceMB + "MB, Violating resource: " + p + ").");
+ }
+ }
+ }
+
+ /**
+ * Recursively explore the given path and enforce the limits for resource
+ * localization. This method assumes that there are no symlinks in the
+ * directory structure.
+ */
+ private void explorePath(Configuration job, Path p,
+ LimitChecker limitChecker, Map<URI, FileStatus> statCache)
+ throws IOException {
+ Path pathWithScheme = p;
+ if (!pathWithScheme.toUri().isAbsolute()) {
+ // the path does not have a scheme, so we assume it is a path from the
+ // local filesystem
+ FileSystem localFs = FileSystem.getLocal(job);
+ pathWithScheme = localFs.makeQualified(p);
+ }
+ FileStatus status = getFileStatus(statCache, job, pathWithScheme);
+ if (status.isDirectory()) {
+ FileStatus[] statusArray =
+ pathWithScheme.getFileSystem(job).listStatus(pathWithScheme);
+ for (FileStatus s : statusArray) {
+ explorePath(job, s.getPath(), limitChecker, statCache);
+ }
+ } else {
+ limitChecker.addFile(pathWithScheme, status.getLen());
+ }
+ }
+
+ @VisibleForTesting
+ FileStatus getFileStatus(Map<URI, FileStatus> statCache,
+ Configuration job, Path p) throws IOException {
+ URI u = p.toUri();
+ FileStatus status = statCache.get(u);
+ if (status == null) {
+ status = p.getFileSystem(job).getFileStatus(p);
+ statCache.put(u, status);
+ }
+ return status;
+ }
+
// copies a file to the jobtracker filesystem and returns the path where it
// was copied to
private Path copyRemoteFiles(Path parentDir, Path originalPath,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 53e4b38..d740747 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -967,6 +967,34 @@ public interface MRJobConfig {
128;
/**
+ * The maximum number of resources a map reduce job is allowed to submit for
+ * localization via files, libjars, archives, and jobjar command line
+ * arguments and through the distributed cache. If set to 0 the limit is
+ * ignored.
+ */
+ String MAX_RESOURCES = "mapreduce.job.cache.limit.max-resources";
+ int MAX_RESOURCES_DEFAULT = 0;
+
+ /**
+ * The maximum size (in MB) a map reduce job is allowed to submit for
+ * localization via files, libjars, archives, and jobjar command line
+ * arguments and through the distributed cache. If set to 0 the limit is
+ * ignored.
+ */
+ String MAX_RESOURCES_MB = "mapreduce.job.cache.limit.max-resources-mb";
+ long MAX_RESOURCES_MB_DEFAULT = 0;
+
+ /**
+ * The maximum size (in MB) of a single resource a map reduce job is allow to
+ * submit for localization via files, libjars, archives, and jobjar command
+ * line arguments and through the distributed cache. If set to 0 the limit is
+ * ignored.
+ */
+ String MAX_SINGLE_RESOURCE_MB =
+ "mapreduce.job.cache.limit.max-single-resource-mb";
+ long MAX_SINGLE_RESOURCE_MB_DEFAULT = 0;
+
+ /**
* Number of OPPORTUNISTIC Containers per 100 containers that will be
* requested by the MRAppMaster. The Default value is 0, which implies all
* maps will be guaranteed. A value of 100 means all maps will be requested
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
index 19470e8..73a0330 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
@@ -54,10 +54,23 @@ public class ClientDistributedCacheManager {
public static void determineTimestampsAndCacheVisibilities(Configuration job)
throws IOException {
Map<URI, FileStatus> statCache = new HashMap<URI, FileStatus>();
+ determineTimestampsAndCacheVisibilities(job, statCache);
+ }
+
+ /**
+ * See ClientDistributedCacheManager#determineTimestampsAndCacheVisibilities(
+ * Configuration).
+ *
+ * @param job Configuration of a job
+ * @param statCache A map containing cached file status objects
+ * @throws IOException if there is a problem with the underlying filesystem
+ */
+ public static void determineTimestampsAndCacheVisibilities(Configuration job,
+ Map<URI, FileStatus> statCache) throws IOException {
determineTimestamps(job, statCache);
determineCacheVisibilities(job, statCache);
}
-
+
/**
* Determines timestamps of files to be cached, and stores those
* in the configuration. This is intended to be used internally by JobClient
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 8ba0ec9..33eece3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1916,6 +1916,36 @@
</property>
<property>
+ <name>mapreduce.job.cache.limit.max-resources</name>
+ <value>0</value>
+ <description>The maximum number of resources a map reduce job is allowed to
+ submit for localization via files, libjars, archives, and jobjar command
+ line arguments and through the distributed cache. If set to 0 the limit is
+ ignored.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.job.cache.limit.max-resources-mb</name>
+ <value>0</value>
+ <description>The maximum size (in MB) a map reduce job is allowed to submit
+ for localization via files, libjars, archives, and jobjar command line
+ arguments and through the distributed cache. If set to 0 the limit is
+ ignored.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.job.cache.limit.max-single-resource-mb</name>
+ <value>0</value>
+ <description>The maximum size (in MB) of a single resource a map reduce job
+ is allow to submit for localization via files, libjars, archives, and
+ jobjar command line arguments and through the distributed cache. If set to
+ 0 the limit is ignored.
+ </description>
+</property>
+
+<property>
<description>
Value of the xframe-options
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
new file mode 100644
index 0000000..36ea57a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * A class for unit testing JobResourceUploader.
+ */
+public class TestJobResourceUploader {
+
+ @Test
+ public void testAllDefaults() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ runLimitsTest(b.build(), true, null);
+ }
+
+ @Test
+ public void testNoLimitsWithResources() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(1);
+ b.setNumOfTmpArchives(10);
+ b.setNumOfTmpFiles(1);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setSizeOfResource(10);
+ runLimitsTest(b.build(), true, null);
+ }
+
+ @Test
+ public void testAtResourceLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(1);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(1);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxResources(6);
+ runLimitsTest(b.build(), true, null);
+ }
+
+ @Test
+ public void testOverResourceLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(1);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(2);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxResources(6);
+ runLimitsTest(b.build(), false, ResourceViolation.NUMBER_OF_RESOURCES);
+ }
+
+ @Test
+ public void testAtResourcesMBLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(1);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(2);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxResourcesMB(7);
+ b.setSizeOfResource(1);
+ runLimitsTest(b.build(), true, null);
+ }
+
+ @Test
+ public void testOverResourcesMBLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(2);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(2);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxResourcesMB(7);
+ b.setSizeOfResource(1);
+ runLimitsTest(b.build(), false, ResourceViolation.TOTAL_RESOURCE_SIZE);
+ }
+
+ @Test
+ public void testAtSingleResourceMBLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(2);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(2);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxSingleResourceMB(1);
+ b.setSizeOfResource(1);
+ runLimitsTest(b.build(), true, null);
+ }
+
+ @Test
+ public void testOverSingleResourceMBLimit() throws IOException {
+ ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder();
+ b.setNumOfDCArchives(1);
+ b.setNumOfDCFiles(2);
+ b.setNumOfTmpArchives(1);
+ b.setNumOfTmpFiles(2);
+ b.setNumOfTmpLibJars(1);
+ b.setJobJar(true);
+ b.setMaxSingleResourceMB(1);
+ b.setSizeOfResource(10);
+ runLimitsTest(b.build(), false, ResourceViolation.SINGLE_RESOURCE_SIZE);
+ }
+
+ private enum ResourceViolation {
+ NUMBER_OF_RESOURCES, TOTAL_RESOURCE_SIZE, SINGLE_RESOURCE_SIZE;
+ }
+
+ private void runLimitsTest(ResourceLimitsConf rlConf,
+ boolean checkShouldSucceed, ResourceViolation violation)
+ throws IOException {
+
+ if (!checkShouldSucceed && violation == null) {
+ Assert.fail("Test is misconfigured. checkShouldSucceed is set to false"
+ + " and a ResourceViolation is not specified.");
+ }
+
+ JobConf conf = setupJobConf(rlConf);
+ JobResourceUploader uploader = new StubedUploader(conf);
+ long configuredSizeOfResourceBytes = rlConf.sizeOfResource * 1024 * 1024;
+ when(mockedStatus.getLen()).thenReturn(configuredSizeOfResourceBytes);
+ when(mockedStatus.isDirectory()).thenReturn(false);
+ Map<URI, FileStatus> statCache = new HashMap<URI, FileStatus>();
+ try {
+ uploader.checkLocalizationLimits(conf,
+ conf.getStringCollection("tmpfiles"),
+ conf.getStringCollection("tmpjars"),
+ conf.getStringCollection("tmparchives"),
+ conf.getJar(), statCache);
+ Assert.assertTrue("Limits check succeeded when it should have failed.",
+ checkShouldSucceed);
+ } catch (IOException e) {
+ if (checkShouldSucceed) {
+ Assert.fail("Limits check failed when it should have succeeded: " + e);
+ }
+ switch (violation) {
+ case NUMBER_OF_RESOURCES:
+ if (!e.getMessage().contains(
+ JobResourceUploader.MAX_RESOURCE_ERR_MSG)) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ case TOTAL_RESOURCE_SIZE:
+ if (!e.getMessage().contains(
+ JobResourceUploader.MAX_TOTAL_RESOURCE_MB_ERR_MSG)) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ case SINGLE_RESOURCE_SIZE:
+ if (!e.getMessage().contains(
+ JobResourceUploader.MAX_SINGLE_RESOURCE_MB_ERR_MSG)) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ default:
+ Assert.fail("Test failed unexpectedly: " + e);
+ break;
+ }
+ }
+ }
+
+ private final FileStatus mockedStatus = mock(FileStatus.class);
+
+ private JobConf setupJobConf(ResourceLimitsConf rlConf) {
+ JobConf conf = new JobConf();
+ conf.setInt(MRJobConfig.MAX_RESOURCES, rlConf.maxResources);
+ conf.setLong(MRJobConfig.MAX_RESOURCES_MB, rlConf.maxResourcesMB);
+ conf.setLong(MRJobConfig.MAX_SINGLE_RESOURCE_MB,
+ rlConf.maxSingleResourceMB);
+
+ conf.set("tmpfiles",
+ buildPathString("file://tmpFiles", rlConf.numOfTmpFiles));
+ conf.set("tmpjars",
+ buildPathString("file://tmpjars", rlConf.numOfTmpLibJars));
+ conf.set("tmparchives",
+ buildPathString("file://tmpArchives", rlConf.numOfTmpArchives));
+ conf.set(MRJobConfig.CACHE_ARCHIVES,
+ buildPathString("file://cacheArchives", rlConf.numOfDCArchives));
+ conf.set(MRJobConfig.CACHE_FILES,
+ buildPathString("file://cacheFiles", rlConf.numOfDCFiles));
+ if (rlConf.jobJar) {
+ conf.setJar("file://jobjar.jar");
+ }
+ return conf;
+ }
+
+ private String buildPathString(String pathPrefix, int numOfPaths) {
+ if (numOfPaths < 1) {
+ return "";
+ } else {
+ StringBuilder b = new StringBuilder();
+ b.append(pathPrefix + 0);
+ for (int i = 1; i < numOfPaths; i++) {
+ b.append("," + pathPrefix + i);
+ }
+ return b.toString();
+ }
+ }
+
+ final static class ResourceLimitsConf {
+ private final int maxResources;
+ private final long maxResourcesMB;
+ private final long maxSingleResourceMB;
+ private final int numOfTmpFiles;
+ private final int numOfTmpArchives;
+ private final int numOfTmpLibJars;
+ private final boolean jobJar;
+ private final int numOfDCFiles;
+ private final int numOfDCArchives;
+ private final long sizeOfResource;
+
+ static final ResourceLimitsConf DEFAULT = new ResourceLimitsConf();
+
+ private ResourceLimitsConf() {
+ this(new Builder());
+ }
+
+ private ResourceLimitsConf(Builder builder) {
+ this.maxResources = builder.maxResources;
+ this.maxResourcesMB = builder.maxResourcesMB;
+ this.maxSingleResourceMB = builder.maxSingleResourceMB;
+ this.numOfTmpFiles = builder.numOfTmpFiles;
+ this.numOfTmpArchives = builder.numOfTmpArchives;
+ this.numOfTmpLibJars = builder.numOfTmpLibJars;
+ this.jobJar = builder.jobJar;
+ this.numOfDCFiles = builder.numOfDCFiles;
+ this.numOfDCArchives = builder.numOfDCArchives;
+ this.sizeOfResource = builder.sizeOfResource;
+ }
+
+ static class Builder {
+ // Defaults
+ private int maxResources = 0;
+ private long maxResourcesMB = 0;
+ private long maxSingleResourceMB = 0;
+ private int numOfTmpFiles = 0;
+ private int numOfTmpArchives = 0;
+ private int numOfTmpLibJars = 0;
+ private boolean jobJar = false;
+ private int numOfDCFiles = 0;
+ private int numOfDCArchives = 0;
+ private long sizeOfResource = 0;
+
+ Builder() {
+ }
+
+ Builder setMaxResources(int max) {
+ this.maxResources = max;
+ return this;
+ }
+
+ Builder setMaxResourcesMB(long max) {
+ this.maxResourcesMB = max;
+ return this;
+ }
+
+ Builder setMaxSingleResourceMB(long max) {
+ this.maxSingleResourceMB = max;
+ return this;
+ }
+
+ Builder setNumOfTmpFiles(int num) {
+ this.numOfTmpFiles = num;
+ return this;
+ }
+
+ Builder setNumOfTmpArchives(int num) {
+ this.numOfTmpArchives = num;
+ return this;
+ }
+
+ Builder setNumOfTmpLibJars(int num) {
+ this.numOfTmpLibJars = num;
+ return this;
+ }
+
+ Builder setJobJar(boolean jar) {
+ this.jobJar = jar;
+ return this;
+ }
+
+ Builder setNumOfDCFiles(int num) {
+ this.numOfDCFiles = num;
+ return this;
+ }
+
+ Builder setNumOfDCArchives(int num) {
+ this.numOfDCArchives = num;
+ return this;
+ }
+
+ Builder setSizeOfResource(long sizeMB) {
+ this.sizeOfResource = sizeMB;
+ return this;
+ }
+
+ ResourceLimitsConf build() {
+ return new ResourceLimitsConf(this);
+ }
+ }
+ }
+
+ class StubedUploader extends JobResourceUploader {
+ StubedUploader(JobConf conf) throws IOException {
+ super(FileSystem.getLocal(conf), false);
+ }
+
+ @Override
+ FileStatus getFileStatus(Map<URI, FileStatus> statCache, Configuration job,
+ Path p) throws IOException {
+ return mockedStatus;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80a7298/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
index 451ec57..32b3a42 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
@@ -138,6 +138,8 @@ public class TestMRJobs {
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
private static final String OUTPUT_ROOT_DIR = "/tmp/" +
TestMRJobs.class.getSimpleName();
+ private static final Path TEST_RESOURCES_DIR = new Path(TEST_ROOT_DIR,
+ "localizedResources");
@BeforeClass
public static void setup() throws IOException {
@@ -173,7 +175,7 @@ public class TestMRJobs {
}
@AfterClass
- public static void tearDown() {
+ public static void tearDown() throws IOException {
if (mrCluster != null) {
mrCluster.stop();
mrCluster = null;
@@ -182,6 +184,10 @@ public class TestMRJobs {
dfsCluster.shutdown();
dfsCluster = null;
}
+ if (localFs.exists(TEST_RESOURCES_DIR)) {
+ // clean up resource directory
+ localFs.delete(TEST_RESOURCES_DIR, true);
+ }
}
@After
@@ -189,6 +195,39 @@ public class TestMRJobs {
numSleepReducers = DEFAULT_REDUCES;
}
+ private static void setupJobResourceDirs() throws IOException {
+ if (localFs.exists(TEST_RESOURCES_DIR)) {
+ // clean up directory
+ localFs.delete(TEST_RESOURCES_DIR, true);
+ }
+
+ localFs.mkdirs(TEST_RESOURCES_DIR);
+ FSDataOutputStream outF1 = null;
+ try {
+ // 10KB file
+ outF1 = localFs.create(new Path(TEST_RESOURCES_DIR, "file1.txt"));
+ outF1.write(new byte[10 * 1024]);
+ } finally {
+ if (outF1 != null) {
+ outF1.close();
+ }
+ }
+ localFs.createNewFile(new Path(TEST_RESOURCES_DIR, "file2.txt"));
+ Path subDir = new Path(TEST_RESOURCES_DIR, "subDir");
+ localFs.mkdirs(subDir);
+ FSDataOutputStream outF3 = null;
+ try {
+ // 1MB (plus 10 Bytes) file
+ outF3 = localFs.create(new Path(subDir, "file3.txt"));
+ outF3.write(new byte[(1 * 1024 * 1024) + 10]);
+ } finally {
+ if (outF3 != null) {
+ outF3.close();
+ }
+ }
+ localFs.createNewFile(new Path(subDir, "file4.txt"));
+ }
+
@Test (timeout = 300000)
public void testSleepJob() throws Exception {
testSleepJobInternal(false);
@@ -199,16 +238,99 @@ public class TestMRJobs {
testSleepJobInternal(true);
}
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalResourceUnderLimit() throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well above what is expected
+ sleepConf.setInt(MRJobConfig.MAX_RESOURCES, 6);
+ sleepConf.setLong(MRJobConfig.MAX_RESOURCES_MB, 6);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, false, true, null);
+ }
+
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalResourceSizeOverLimit() throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well below what is expected
+ sleepConf.setLong(MRJobConfig.MAX_RESOURCES_MB, 1);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, false, false,
+ ResourceViolation.TOTAL_RESOURCE_SIZE);
+ }
+
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalResourceNumberOverLimit() throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well below what is expected
+ sleepConf.setInt(MRJobConfig.MAX_RESOURCES, 1);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, false, false,
+ ResourceViolation.NUMBER_OF_RESOURCES);
+ }
+
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalResourceCheckAndRemoteJar()
+ throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well above what is expected
+ sleepConf.setInt(MRJobConfig.MAX_RESOURCES, 6);
+ sleepConf.setLong(MRJobConfig.MAX_RESOURCES_MB, 6);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, true, true, null);
+ }
+
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalIndividualResourceOverLimit()
+ throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well below what is expected
+ sleepConf.setInt(MRJobConfig.MAX_SINGLE_RESOURCE_MB, 1);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, false, false,
+ ResourceViolation.SINGLE_RESOURCE_SIZE);
+ }
+
+ @Test(timeout = 300000)
+ public void testSleepJobWithLocalIndividualResourceUnderLimit()
+ throws Exception {
+ Configuration sleepConf = new Configuration(mrCluster.getConfig());
+ // set limits to well below what is expected
+ sleepConf.setInt(MRJobConfig.MAX_SINGLE_RESOURCE_MB, 2);
+ setupJobResourceDirs();
+ sleepConf.set("tmpfiles", TEST_RESOURCES_DIR.toString());
+ testSleepJobInternal(sleepConf, false, true, null);
+ }
+
private void testSleepJobInternal(boolean useRemoteJar) throws Exception {
+ testSleepJobInternal(new Configuration(mrCluster.getConfig()),
+ useRemoteJar, true, null);
+ }
+
+ private enum ResourceViolation {
+ NUMBER_OF_RESOURCES, TOTAL_RESOURCE_SIZE, SINGLE_RESOURCE_SIZE;
+ }
+
+ private void testSleepJobInternal(Configuration sleepConf,
+ boolean useRemoteJar, boolean jobSubmissionShouldSucceed,
+ ResourceViolation violation) throws Exception {
LOG.info("\n\n\nStarting testSleepJob: useRemoteJar=" + useRemoteJar);
+ if (!jobSubmissionShouldSucceed && violation == null) {
+ Assert.fail("Test is misconfigured. jobSubmissionShouldSucceed is set"
+ + " to false and a ResourceViolation is not specified.");
+ }
+
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ " not found. Not running test.");
return;
}
- Configuration sleepConf = new Configuration(mrCluster.getConfig());
// set master address to local to test that local mode applied iff framework == local
sleepConf.set(MRConfig.MASTER_ADDRESS, "local");
@@ -229,7 +351,45 @@ public class TestMRJobs {
job.setJarByClass(SleepJob.class);
}
job.setMaxMapAttempts(1); // speed up failures
- job.submit();
+ try {
+ job.submit();
+ Assert.assertTrue("JobSubmission succeeded when it should have failed.",
+ jobSubmissionShouldSucceed);
+ } catch (IOException e) {
+ if (jobSubmissionShouldSucceed) {
+ Assert
+ .fail("Job submission failed when it should have succeeded: " + e);
+ }
+ switch (violation) {
+ case NUMBER_OF_RESOURCES:
+ if (!e.getMessage().contains(
+ "This job has exceeded the maximum number of"
+ + " submitted resources")) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ case TOTAL_RESOURCE_SIZE:
+ if (!e.getMessage().contains(
+ "This job has exceeded the maximum size of submitted resources")) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ case SINGLE_RESOURCE_SIZE:
+ if (!e.getMessage().contains(
+ "This job has exceeded the maximum size of a single submitted")) {
+ Assert.fail("Test failed unexpectedly: " + e);
+ }
+ break;
+
+ default:
+ Assert.fail("Test failed unexpectedly: " + e);
+ break;
+ }
+ // we are done with the test (job submission failed)
+ return;
+ }
String trackingUrl = job.getTrackingURL();
String jobId = job.getJobID().toString();
boolean succeeded = job.waitForCompletion(true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-10580. DiskBalancer: Make use
of unused methods in GreedyPlanner to print debug info. Contributed by Yiqun
Lin
Posted by cn...@apache.org.
HDFS-10580. DiskBalancer: Make use of unused methods in GreedyPlanner to print debug info. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bed69d18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bed69d18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bed69d18
Branch: refs/heads/HADOOP-13345
Commit: bed69d18e6c84583cbe5fd765f068d9faa807617
Parents: 2424911
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Aug 15 12:40:29 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Aug 15 12:40:29 2016 -0700
----------------------------------------------------------------------
.../diskbalancer/planner/GreedyPlanner.java | 45 +++++++++++---------
1 file changed, 25 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bed69d18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
index 0df9843..fb83eeb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
@@ -158,6 +158,7 @@ public class GreedyPlanner implements Planner {
// since the volume data changed , we need to recompute the DataDensity.
currentSet.computeVolumeDataDensity();
+ printQueue(currentSet.getSortedQueue());
}
/**
@@ -184,7 +185,7 @@ public class GreedyPlanner implements Planner {
if (maxLowVolumeCanReceive <= 0) {
LOG.debug("{} Skipping disk from computation. Maximum data size " +
"achieved.", lowVolume.getPath());
- lowVolume.setSkip(true);
+ skipVolume(currentSet, lowVolume);
}
long maxHighVolumeCanGive = highVolume.getUsed() -
@@ -195,7 +196,7 @@ public class GreedyPlanner implements Planner {
if (maxHighVolumeCanGive <= 0) {
LOG.debug(" {} Skipping disk from computation. Minimum data size " +
"achieved.", highVolume.getPath());
- highVolume.setSkip(true);
+ skipVolume(currentSet, highVolume);
}
@@ -219,16 +220,19 @@ public class GreedyPlanner implements Planner {
*/
private void skipVolume(DiskBalancerVolumeSet currentSet,
DiskBalancerVolume volume) {
-
- String message = String.format(
- "Skipping volume. Volume : %s " +
- "Type : %s Target " +
- "Number of bytes : %f lowVolume dfsUsed : %d. Skipping this " +
- "volume from all future balancing calls.", volume.getPath(),
- volume.getStorageType(),
- currentSet.getIdealUsed() * volume.getCapacity(), volume.getUsed());
+ if (LOG.isDebugEnabled()) {
+ String message =
+ String.format(
+ "Skipping volume. Volume : %s " +
+ "Type : %s Target " +
+ "Number of bytes : %f lowVolume dfsUsed : %d. Skipping this " +
+ "volume from all future balancing calls.", volume.getPath(),
+ volume.getStorageType(),
+ currentSet.getIdealUsed() * volume.getCapacity(),
+ volume.getUsed());
+ LOG.debug(message);
+ }
volume.setSkip(true);
- LOG.debug(message);
}
// Removes all volumes which are part of the volumeSet but skip flag is set.
@@ -242,6 +246,7 @@ public class GreedyPlanner implements Planner {
}
}
currentSet.computeVolumeDataDensity();
+ printQueue(currentSet.getSortedQueue());
}
/**
@@ -251,14 +256,14 @@ public class GreedyPlanner implements Planner {
* @param queue - Queue
*/
private void printQueue(TreeSet<DiskBalancerVolume> queue) {
- String format = String.format("First Volume : %s, DataDensity : %f",
- queue.first().getPath(),
- queue.first().getVolumeDataDensity());
- LOG.info(format);
-
- format = String
- .format("Last Volume : %s, DataDensity : %f%n", queue.last().getPath(),
- queue.last().getVolumeDataDensity());
- LOG.info(format);
+ if (LOG.isDebugEnabled()) {
+ String format =
+ String.format(
+ "First Volume : %s, DataDensity : %f, " +
+ "Last Volume : %s, DataDensity : %f",
+ queue.first().getPath(), queue.first().getVolumeDataDensity(),
+ queue.last().getPath(), queue.last().getVolumeDataDensity());
+ LOG.debug(format);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDFS-10678. Documenting
NNThroughputBenchmark tool. (Contributed by Mingliang Liu)
Posted by cn...@apache.org.
HDFS-10678. Documenting NNThroughputBenchmark tool. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/382d6152
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/382d6152
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/382d6152
Branch: refs/heads/HADOOP-13345
Commit: 382d6152602339fe58169b2918ec74e7a7cd5581
Parents: 4bcbef3
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:22:14 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:22:14 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/Benchmarking.md | 106 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 32 +-----
hadoop-project/src/site/site.xml | 1 +
3 files changed, 110 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/382d6152/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md b/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
new file mode 100644
index 0000000..678dcee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
@@ -0,0 +1,106 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Hadoop Benchmarking
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+This page is to discuss benchmarking Hadoop using tools it provides.
+
+## NNThroughputBenchmark
+
+### Overview
+
+**NNThroughputBenchmark**, as its name indicates, is a name-node throughput benchmark, which runs a series of client threads on a single node against a name-node. If no name-node is configured, it will firstly start a name-node in the same process (_standalone mode_), in which case each client repetitively performs the same operation by directly calling the respective name-node methods. Otherwise, the benchmark will perform the operations against a remote name-node via client protocol RPCs (_remote mode_). Either way, all clients are running locally in a single process rather than remotely across different nodes. The reason is to avoid communication overhead caused by RPC connections and serialization, and thus reveal the upper bound of pure name-node performance.
+
+The benchmark first generates inputs for each thread so that the input generation overhead does not effect the resulting statistics. The number of operations performed by threads is practically the same. Precisely, the difference between the number of operations performed by any two threads does not exceed 1. Then the benchmark executes the specified number of operations using the specified number of threads and outputs the resulting stats by measuring the number of operations performed by the name-node per second.
+
+### Commands
+
+The general command line syntax is:
+
+`hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark [genericOptions] [commandOptions]`
+
+#### Generic Options
+
+This benchmark honors the [Hadoop command-line Generic Options](CommandsManual.html#Generic_Options) to alter its behavior. The benchmark, as other tools, will rely on the `fs.defaultFS` config, which is overridable by `-fs` command option, to run standalone mode or remote mode. If the `fs.defaultFS` scheme is not specified or is `file` (local), the benchmark will run in _standalone mode_. Specially, the _remote_ name-node config `dfs.namenode.fs-limits.min-block-size` should be set as 16 while in _standalone mode_ the benchmark turns off minimum block size verification for its internal name-node.
+
+#### Command Options
+
+The following are all supported command options:
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+|`-op` | Specify the operation. This option must be provided and should be the first option. |
+|`-logLevel` | Specify the logging level when the benchmark runs. The default logging level is ERROR. |
+|`-UGCacheRefreshCount` | After every specified number of operations, the benchmark purges the name-node's user group cache. By default the refresh is never called. |
+|`-keepResults` | If specified, do not clean up the name-space after execution. By default the name-space will be removed after test. |
+
+##### Operations Supported
+
+Following are all the operations supported along with their respective operation-specific parameters (all optional) and default values.
+
+| OPERATION\_OPTION | Operation-specific parameters |
+|:---- |:---- |
+|`all` | _options for other operations_ |
+|`create` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-close`] |
+|`mkdirs` | [`-threads 3`] [`-dirs 10`] [`-dirsPerDir 2`] |
+|`open` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`delete` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`fileStatus` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`rename` | [`-threads 3`] [`-files 10`] [`-filesPerDir 4`] [`-useExisting`] |
+|`blockReport` | [`-datanodes 10`] [`-reports 30`] [`-blocksPerReport 100`] [`-blocksPerFile 10`] |
+|`replication` | [`-datanodes 10`] [`-nodesToDecommission 1`] [`-nodeReplicationLimit 100`] [`-totalBlocks 100`] [`-replication 3`] |
+|`clean` | N/A |
+
+##### Operation Options
+
+When running benchmarks with the above operation(s), please provide operation-specific parameters illustrated as following.
+
+| OPERATION\_SPECIFIC\_OPTION | Description |
+|:---- |:---- |
+|`-threads` | Number of total threads to run the respective operation. |
+|`-files` | Number of total files for the respective operation. |
+|`-dirs` | Number of total directories for the respective operation. |
+|`-filesPerDir` | Number of files per directory. |
+|`-close` | Close the files after creation. |
+|`-dirsPerDir` | Number of directories per directory. |
+|`-useExisting` | If specified, do not recreate the name-space, use existing data. |
+|`-datanodes` | Total number of simulated data-nodes. |
+|`-reports` | Total number of block reports to send. |
+|`-blocksPerReport` | Number of blocks per report. |
+|`-blocksPerFile` | Number of blocks per file. |
+|`-nodesToDecommission` | Total number of simulated data-nodes to decommission. |
+|`-nodeReplicationLimit` | The maximum number of outgoing replication streams for a data-node. |
+|`-totalBlocks` | Number of total blocks to operate. |
+|`-replication` | Replication factor. Will be adjusted to number of data-nodes if it is larger than that. |
+
+### Reports
+
+The benchmark measures the number of operations performed by the name-node per second. Specifically, for each operation tested, it reports the total running time in seconds (_Elapsed Time_), operation throughput (_Ops per sec_), and average time for the operations (_Average Time_). The higher, the better.
+
+Following is a sample reports by running following commands that opens 100K files with 1K threads against a remote name-node. See [HDFS scalability: the limits to growth](https://www.usenix.org/legacy/publications/login/2010-04/openpdfs/shvachko.pdf) for real-world benchmark stats.
+```
+$ hadoop org.apache.hadoop.hdfs.server.namenode.NNThroughputBenchmark -fs hdfs://nameservice:9000 -op open -threads 1000 -files 100000
+
+--- open inputs ---
+nrFiles = 100000
+nrThreads = 1000
+nrFilesPerDir = 4
+--- open stats ---
+# operations: 100000
+Elapsed Time: 9510
+ Ops per sec: 10515.247108307045
+Average Time: 90
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/382d6152/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index efd731e..be2a678 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -95,36 +95,10 @@ import org.apache.log4j.LogManager;
* except for the name-node. Each operation is executed
* by calling directly the respective name-node method.
* The name-node here is real all other components are simulated.
- *
- * This benchmark supports
- * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options">
- * standard command-line options</a>. If you use remote namenode by <tt>-fs</tt>
- * option, its config <tt>dfs.namenode.fs-limits.min-block-size</tt> should be
- * set as 16.
*
- * Command line arguments for the benchmark include:
- * <ol>
- * <li>total number of operations to be performed,</li>
- * <li>number of threads to run these operations,</li>
- * <li>followed by operation specific input parameters.</li>
- * <li>-logLevel L specifies the logging level when the benchmark runs.
- * The default logging level is {@link Level#ERROR}.</li>
- * <li>-UGCacheRefreshCount G will cause the benchmark to call
- * {@link NameNodeRpcServer#refreshUserToGroupsMappings} after
- * every G operations, which purges the name-node's user group cache.
- * By default the refresh is never called.</li>
- * <li>-keepResults do not clean up the name-space after execution.</li>
- * <li>-useExisting do not recreate the name-space, use existing data.</li>
- * </ol>
- *
- * The benchmark first generates inputs for each thread so that the
- * input generation overhead does not effect the resulting statistics.
- * The number of operations performed by threads is practically the same.
- * Precisely, the difference between the number of operations
- * performed by any two threads does not exceed 1.
- *
- * Then the benchmark executes the specified number of operations using
- * the specified number of threads and outputs the resulting stats.
+ * For usage, please see <a href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Benchmarking.html#NNThroughputBenchmark">the documentation</a>.
+ * Meanwhile, if you change the usage of this program, please also update the
+ * documentation accordingly.
*/
public class NNThroughputBenchmark implements Tool {
private static final Log LOG = LogFactory.getLog(NNThroughputBenchmark.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/382d6152/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index de09016..9fa1469 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -169,6 +169,7 @@
<item name="GridMix" href="hadoop-gridmix/GridMix.html"/>
<item name="Rumen" href="hadoop-rumen/Rumen.html"/>
<item name="Scheduler Load Simulator" href="hadoop-sls/SchedulerLoadSimulator.html"/>
+ <item name="Hadoop Benchmarking" href="hadoop-project-dist/hadoop-common/Benchmarking.html"/>
</menu>
<menu name="Reference" inherit="top">
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-5492.
TestSubmitApplicationWithRMHA is failing sporadically during precommit
builds. Contributed by Vrushali C.
Posted by cn...@apache.org.
YARN-5492. TestSubmitApplicationWithRMHA is failing sporadically during precommit builds. Contributed by Vrushali C.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5199db38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5199db38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5199db38
Branch: refs/heads/HADOOP-13345
Commit: 5199db387d59f7233a0e52ac298df31e8ed8af20
Parents: 666ad0b
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Aug 11 11:50:46 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Thu Aug 11 11:50:46 2016 +0530
----------------------------------------------------------------------
.../server/resourcemanager/TestSubmitApplicationWithRMHA.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5199db38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java
index 4cceddd..4658d60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestSubmitApplicationWithRMHA.java
@@ -224,7 +224,7 @@ public class TestSubmitApplicationWithRMHA extends RMHATestBase{
// during SubmitApplication Call:
// 1) RMStateStore already saved the ApplicationState when failover happens
// 2) RMStateStore did not save the ApplicationState when failover happens
- @Test (timeout = 5000)
+ @Test (timeout = 50000)
public void
testHandleRMHADuringSubmitApplicationCallWithSavedApplicationState()
throws Exception {
@@ -260,7 +260,7 @@ public class TestSubmitApplicationWithRMHA extends RMHATestBase{
Assert.assertEquals(app1.getApplicationId(), app0.getApplicationId());
}
- @Test (timeout = 5000)
+ @Test (timeout = 50000)
public void
testHandleRMHADuringSubmitApplicationCallWithoutSavedApplicationState()
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HADOOP-13405 doc for
fs.s3a.acl.default indicates incorrect values. Contributed by Shen Yinjie
Posted by cn...@apache.org.
HADOOP-13405 doc for fs.s3a.acl.default indicates incorrect values. Contributed by Shen Yinjie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/040c185d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/040c185d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/040c185d
Branch: refs/heads/HADOOP-13345
Commit: 040c185d624a18627d23cedb12bf91a950ada2fc
Parents: 913a895
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 18 14:35:26 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 18 14:36:55 2016 +0100
----------------------------------------------------------------------
.../hadoop-common/src/main/resources/core-default.xml | 6 +++---
.../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 4 ++--
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 +++---
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e78795c..dd8f9af 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1039,9 +1039,9 @@
<property>
<name>fs.s3a.acl.default</name>
- <description>Set a canned ACL for newly created and copied objects. Value may be private,
- public-read, public-read-write, authenticated-read, log-delivery-write,
- bucket-owner-read, or bucket-owner-full-control.</description>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 612b648..6b8f9f5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -128,8 +128,8 @@ public final class Constants {
public static final String FAST_BUFFER_SIZE = "fs.s3a.fast.buffer.size";
public static final int DEFAULT_FAST_BUFFER_SIZE = 1048576; //1MB
- // private | public-read | public-read-write | authenticated-read |
- // log-delivery-write | bucket-owner-read | bucket-owner-full-control
+ // Private | PublicRead | PublicReadWrite | AuthenticatedRead |
+ // LogDeliveryWrite | BucketOwnerRead | BucketOwnerFullControl
public static final String CANNED_ACL = "fs.s3a.acl.default";
public static final String DEFAULT_CANNED_ACL = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/040c185d/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index cb1df83..d459fb5 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -554,9 +554,9 @@ this capability.
<property>
<name>fs.s3a.acl.default</name>
- <description>Set a canned ACL for newly created and copied objects. Value may be private,
- public-read, public-read-write, authenticated-read, log-delivery-write,
- bucket-owner-read, or bucket-owner-full-control.</description>
+ <description>Set a canned ACL for newly created and copied objects. Value may be Private,
+ PublicRead, PublicReadWrite, AuthenticatedRead, LogDeliveryWrite, BucketOwnerRead,
+ or BucketOwnerFullControl.</description>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDFS-10731.
FSDirectory#verifyMaxDirItems does not log path name. Contributed by Wei-Chiu
Chuang.
Posted by cn...@apache.org.
HDFS-10731. FSDirectory#verifyMaxDirItems does not log path name. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9019606b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9019606b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9019606b
Branch: refs/heads/HADOOP-13345
Commit: 9019606b69bfb7019c8642b6cbcbb93645cc19e3
Parents: b5af9be
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Aug 11 14:43:48 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Fri Aug 12 05:58:13 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/protocol/FSLimitException.java | 4 +++-
.../java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java | 4 ++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9019606b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FSLimitException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FSLimitException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FSLimitException.java
index 2ee11f0..347d892 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FSLimitException.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/FSLimitException.java
@@ -87,8 +87,10 @@ public abstract class FSLimitException extends QuotaExceededException {
super(msg);
}
- public MaxDirectoryItemsExceededException(long quota, long count) {
+ public MaxDirectoryItemsExceededException(String path, long quota,
+ long count) {
super(quota, count);
+ setPathName(path);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9019606b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index b71b8d6..11a7899 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -991,9 +991,9 @@ public class FSDirectory implements Closeable {
final int count = parent.getChildrenList(CURRENT_STATE_ID).size();
if (count >= maxDirItems) {
final MaxDirectoryItemsExceededException e
- = new MaxDirectoryItemsExceededException(maxDirItems, count);
+ = new MaxDirectoryItemsExceededException(parentPath, maxDirItems,
+ count);
if (namesystem.isImageLoaded()) {
- e.setPathName(parentPath);
throw e;
} else {
// Do not throw if edits log is still being processed
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-4676. Automatic and
Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel
Zhi. (cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Posted by cn...@apache.org.
YARN-4676. Automatic and Asynchronous Decommissioning Nodes Status Tracking. Contributed by Diniel Zhi.
(cherry picked from commit d464483bf7f0b3e3be3ba32cd6c3eee546747ab5)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0da69c32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0da69c32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0da69c32
Branch: refs/heads/HADOOP-13345
Commit: 0da69c324dee9baab0f0b9700db1cc5b623f8421
Parents: 040c185
Author: Junping Du <ju...@apache.org>
Authored: Thu Aug 18 07:23:29 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu Aug 18 07:27:23 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/HostsFileReader.java | 111 ++++-
.../apache/hadoop/util/TestHostsFileReader.java | 64 ++-
hadoop-project/src/site/site.xml | 1 +
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 5 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +
.../protocolrecords/RefreshNodesRequest.java | 26 +-
..._server_resourcemanager_service_protos.proto | 1 +
.../hadoop/yarn/client/cli/RMAdminCLI.java | 166 ++++---
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 24 +-
.../impl/pb/RefreshNodesRequestPBImpl.java | 17 +-
.../src/main/resources/yarn-default.xml | 18 +
.../server/resourcemanager/AdminService.java | 3 +-
.../DecommissioningNodesWatcher.java | 439 +++++++++++++++++++
.../resourcemanager/NodesListManager.java | 166 +++++--
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 19 +
.../server/resourcemanager/rmnode/RMNode.java | 6 +
.../rmnode/RMNodeDecommissioningEvent.java | 41 ++
.../resourcemanager/rmnode/RMNodeImpl.java | 54 ++-
.../webapp/dao/ClusterMetricsInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 5 +
.../yarn/server/resourcemanager/MockRM.java | 11 +-
.../TestDecommissioningNodesWatcher.java | 131 ++++++
.../resourcemanager/TestRMNodeTransitions.java | 11 -
.../TestResourceTrackerService.java | 199 +++++++--
.../resourcetracker/TestNMReconnect.java | 2 -
.../src/site/markdown/YarnCommands.md | 2 +-
28 files changed, 1326 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index 1cba426..2ef1ead 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -21,16 +21,27 @@ package org.apache.hadoop.util;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.Set;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
// Keeps track of which datanodes/tasktrackers are allowed to connect to the
// namenode/jobtracker.
@@ -38,7 +49,9 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
public class HostsFileReader {
private Set<String> includes;
- private Set<String> excludes;
+ // exclude host list with optional timeout.
+ // If the value is null, it indicates default timeout.
+ private Map<String, Integer> excludes;
private String includesFile;
private String excludesFile;
private WriteLock writeLock;
@@ -49,7 +62,7 @@ public class HostsFileReader {
public HostsFileReader(String inFile,
String exFile) throws IOException {
includes = new HashSet<String>();
- excludes = new HashSet<String>();
+ excludes = new HashMap<String, Integer>();
includesFile = inFile;
excludesFile = exFile;
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
@@ -62,7 +75,7 @@ public class HostsFileReader {
public HostsFileReader(String includesFile, InputStream inFileInputStream,
String excludesFile, InputStream exFileInputStream) throws IOException {
includes = new HashSet<String>();
- excludes = new HashSet<String>();
+ excludes = new HashMap<String, Integer>();
this.includesFile = includesFile;
this.excludesFile = excludesFile;
ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
@@ -121,6 +134,73 @@ public class HostsFileReader {
}
}
+ public static void readFileToMap(String type,
+ String filename, Map<String, Integer> map) throws IOException {
+ File file = new File(filename);
+ FileInputStream fis = new FileInputStream(file);
+ readFileToMapWithFileInputStream(type, filename, fis, map);
+ }
+
+ public static void readFileToMapWithFileInputStream(String type,
+ String filename, InputStream inputStream, Map<String, Integer> map)
+ throws IOException {
+ // The input file could be either simple text or XML.
+ boolean xmlInput = filename.toLowerCase().endsWith(".xml");
+ if (xmlInput) {
+ readXmlFileToMapWithFileInputStream(type, filename, inputStream, map);
+ } else {
+ HashSet<String> nodes = new HashSet<String>();
+ readFileToSetWithFileInputStream(type, filename, inputStream, nodes);
+ for (String node : nodes) {
+ map.put(node, null);
+ }
+ }
+ }
+
+ public static void readXmlFileToMapWithFileInputStream(String type,
+ String filename, InputStream fileInputStream, Map<String, Integer> map)
+ throws IOException {
+ Document dom;
+ DocumentBuilderFactory builder = DocumentBuilderFactory.newInstance();
+ try {
+ DocumentBuilder db = builder.newDocumentBuilder();
+ dom = db.parse(fileInputStream);
+ // Examples:
+ // <host><name>host1</name></host>
+ // <host><name>host2</name><timeout>123</timeout></host>
+ // <host><name>host3</name><timeout>-1</timeout></host>
+ // <host><name>host4, host5,host6</name><timeout>1800</timeout></host>
+ Element doc = dom.getDocumentElement();
+ NodeList nodes = doc.getElementsByTagName("host");
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Node node = nodes.item(i);
+ if (node.getNodeType() == Node.ELEMENT_NODE) {
+ Element e= (Element) node;
+ // Support both single host and comma-separated list of hosts.
+ String v = readFirstTagValue(e, "name");
+ String[] hosts = StringUtils.getTrimmedStrings(v);
+ String str = readFirstTagValue(e, "timeout");
+ Integer timeout = (str == null)? null : Integer.parseInt(str);
+ for (String host : hosts) {
+ map.put(host, timeout);
+ LOG.info("Adding a node \"" + host + "\" to the list of "
+ + type + " hosts from " + filename);
+ }
+ }
+ }
+ } catch (IOException|SAXException|ParserConfigurationException e) {
+ LOG.fatal("error parsing " + filename, e);
+ throw new RuntimeException(e);
+ } finally {
+ fileInputStream.close();
+ }
+ }
+
+ static String readFirstTagValue(Element e, String tag) {
+ NodeList nodes = e.getElementsByTagName(tag);
+ return (nodes.getLength() == 0)? null : nodes.item(0).getTextContent();
+ }
+
public void refresh(String includeFiles, String excludeFiles)
throws IOException {
LOG.info("Refreshing hosts (include/exclude) list");
@@ -129,7 +209,7 @@ public class HostsFileReader {
// update instance variables
updateFileNames(includeFiles, excludeFiles);
Set<String> newIncludes = new HashSet<String>();
- Set<String> newExcludes = new HashSet<String>();
+ Map<String, Integer> newExcludes = new HashMap<String, Integer>();
boolean switchIncludes = false;
boolean switchExcludes = false;
if (includeFiles != null && !includeFiles.isEmpty()) {
@@ -137,7 +217,7 @@ public class HostsFileReader {
switchIncludes = true;
}
if (excludeFiles != null && !excludeFiles.isEmpty()) {
- readFileToSet("excluded", excludeFiles, newExcludes);
+ readFileToMap("excluded", excludeFiles, newExcludes);
switchExcludes = true;
}
@@ -161,7 +241,7 @@ public class HostsFileReader {
this.writeLock.lock();
try {
Set<String> newIncludes = new HashSet<String>();
- Set<String> newExcludes = new HashSet<String>();
+ Map<String, Integer> newExcludes = new HashMap<String, Integer>();
boolean switchIncludes = false;
boolean switchExcludes = false;
if (inFileInputStream != null) {
@@ -170,7 +250,7 @@ public class HostsFileReader {
switchIncludes = true;
}
if (exFileInputStream != null) {
- readFileToSetWithFileInputStream("excluded", excludesFile,
+ readFileToMapWithFileInputStream("excluded", excludesFile,
exFileInputStream, newExcludes);
switchExcludes = true;
}
@@ -199,7 +279,7 @@ public class HostsFileReader {
public Set<String> getExcludedHosts() {
this.readLock.lock();
try {
- return excludes;
+ return excludes.keySet();
} finally {
this.readLock.unlock();
}
@@ -209,7 +289,18 @@ public class HostsFileReader {
this.readLock.lock();
try {
includes.addAll(this.includes);
- excludes.addAll(this.excludes);
+ excludes.addAll(this.excludes.keySet());
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ public void getHostDetails(Set<String> includeHosts,
+ Map<String, Integer> excludeHosts) {
+ this.readLock.lock();
+ try {
+ includeHosts.addAll(this.includes);
+ excludeHosts.putAll(this.excludes);
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
index 8015f7a..5766591 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestHostsFileReader.java
@@ -20,16 +20,19 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
+import java.util.Map;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.*;
+
import static org.junit.Assert.*;
/*
* Test for HostsFileReader.java
- *
+ *
*/
public class TestHostsFileReader {
@@ -39,6 +42,7 @@ public class TestHostsFileReader {
File INCLUDES_FILE = new File(HOSTS_TEST_DIR, "dfs.include");
String excludesFile = HOSTS_TEST_DIR + "/dfs.exclude";
String includesFile = HOSTS_TEST_DIR + "/dfs.include";
+ private String excludesXmlFile = HOSTS_TEST_DIR + "/dfs.exclude.xml";
@Before
public void setUp() throws Exception {
@@ -288,4 +292,62 @@ public class TestHostsFileReader {
assertFalse(hfp.getExcludedHosts().contains("somehost5"));
}
+
+ /*
+ * Test if timeout values are provided in HostFile
+ */
+ @Test
+ public void testHostFileReaderWithTimeout() throws Exception {
+ FileWriter efw = new FileWriter(excludesXmlFile);
+ FileWriter ifw = new FileWriter(includesFile);
+
+ efw.write("<?xml version=\"1.0\"?>\n");
+ efw.write("<!-- yarn.nodes.exclude -->\n");
+ efw.write("<hosts>\n");
+ efw.write("<host><name>host1</name></host>\n");
+ efw.write("<host><name>host2</name><timeout>123</timeout></host>\n");
+ efw.write("<host><name>host3</name><timeout>-1</timeout></host>\n");
+ efw.write("<host><name>10000</name></host>\n");
+ efw.write("<host><name>10001</name><timeout>123</timeout></host>\n");
+ efw.write("<host><name>10002</name><timeout>-1</timeout></host>\n");
+ efw.write("<host><name>host4,host5, host6</name>" +
+ "<timeout>1800</timeout></host>\n");
+ efw.write("</hosts>\n");
+ efw.close();
+
+ ifw.write("#Hosts-in-DFS\n");
+ ifw.write(" \n");
+ ifw.write(" somehost \t somehost2 \n somehost4");
+ ifw.write(" somehost3 \t # somehost5");
+ ifw.close();
+
+ HostsFileReader hfp = new HostsFileReader(includesFile, excludesXmlFile);
+
+ int includesLen = hfp.getHosts().size();
+ int excludesLen = hfp.getExcludedHosts().size();
+ assertEquals(4, includesLen);
+ assertEquals(9, excludesLen);
+
+ Set<String> includes = new HashSet<String>();
+ Map<String, Integer> excludes = new HashMap<String, Integer>();
+ hfp.getHostDetails(includes, excludes);
+ assertTrue(excludes.containsKey("host1"));
+ assertTrue(excludes.containsKey("host2"));
+ assertTrue(excludes.containsKey("host3"));
+ assertTrue(excludes.containsKey("10000"));
+ assertTrue(excludes.containsKey("10001"));
+ assertTrue(excludes.containsKey("10002"));
+ assertTrue(excludes.containsKey("host4"));
+ assertTrue(excludes.containsKey("host5"));
+ assertTrue(excludes.containsKey("host6"));
+ assertTrue(excludes.get("host1") == null);
+ assertTrue(excludes.get("host2") == 123);
+ assertTrue(excludes.get("host3") == -1);
+ assertTrue(excludes.get("10000") == null);
+ assertTrue(excludes.get("10001") == 123);
+ assertTrue(excludes.get("10002") == -1);
+ assertTrue(excludes.get("host4") == 1800);
+ assertTrue(excludes.get("host5") == 1800);
+ assertTrue(excludes.get("host6") == 1800);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 9fa1469..0d87973 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -136,6 +136,7 @@
<item name="Secure Containers" href="hadoop-yarn/hadoop-yarn-site/SecureContainer.html"/>
<item name="Registry" href="hadoop-yarn/hadoop-yarn-site/registry/index.html"/>
<item name="Reservation System" href="hadoop-yarn/hadoop-yarn-site/ReservationSystem.html"/>
+ <item name="Graceful Decommission" href="hadoop-yarn/hadoop-yarn-site/GracefulDecommission.html"/>
</menu>
<menu name="YARN REST APIs" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index bd737bd..c598aa0 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -213,6 +213,11 @@ public class NodeInfo {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 5048978..6d0ffbd 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -202,4 +202,9 @@ public class RMNodeWrapper implements RMNode {
@Override
public void setUntrackedTimeStamp(long timeStamp) {
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8899ccd..770d139 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -795,6 +795,20 @@ public class YarnConfiguration extends Configuration {
*/
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
+ /**
+ * Timeout in seconds for YARN node graceful decommission.
+ * This is the maximal time to wait for running containers and applications
+ * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.
+ */
+ public static final String RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT =
+ RM_PREFIX + "nodemanager-graceful-decommission-timeout-secs";
+ public static final int DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT = 3600;
+
+ public static final String RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL =
+ RM_PREFIX + "decommissioning-nodes-watcher.poll-interval-secs";
+ public static final int
+ DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL = 20;
+
////////////////////////////////
// Node Manager Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
index 0333c3b..732d98e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RefreshNodesRequest.java
@@ -43,6 +43,16 @@ public abstract class RefreshNodesRequest {
return request;
}
+ @Private
+ @Unstable
+ public static RefreshNodesRequest newInstance(
+ DecommissionType decommissionType, Integer timeout) {
+ RefreshNodesRequest request = Records.newRecord(RefreshNodesRequest.class);
+ request.setDecommissionType(decommissionType);
+ request.setDecommissionTimeout(timeout);
+ return request;
+ }
+
/**
* Set the DecommissionType
*
@@ -56,4 +66,18 @@ public abstract class RefreshNodesRequest {
* @return decommissionType
*/
public abstract DecommissionType getDecommissionType();
-}
+
+ /**
+ * Set the DecommissionTimeout.
+ *
+ * @param timeout graceful decommission timeout in seconds
+ */
+ public abstract void setDecommissionTimeout(Integer timeout);
+
+ /**
+ * Get the DecommissionTimeout.
+ *
+ * @return decommissionTimeout
+ */
+ public abstract Integer getDecommissionTimeout();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index eaf658f..b9f30db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -37,6 +37,7 @@ message RefreshQueuesResponseProto {
message RefreshNodesRequestProto {
optional DecommissionTypeProto decommissionType = 1 [default = NORMAL];
+ optional int32 decommissionTimeout = 2;
}
message RefreshNodesResponseProto {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 4aa3a14..fcb9b74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -99,10 +99,10 @@ public class RMAdminCLI extends HAAdmin {
"properties. \n\t\tResourceManager will reload the " +
"mapred-queues configuration file."))
.put("-refreshNodes",
- new UsageInfo("[-g [timeout in seconds] -client|server]",
+ new UsageInfo("[-g|graceful [timeout in seconds] -client|server]",
"Refresh the hosts information at the ResourceManager. Here "
- + "[-g [timeout in seconds] -client|server] is optional, if we "
- + "specify the timeout then ResourceManager will wait for "
+ + "[-g|graceful [timeout in seconds] -client|server] is optional,"
+ + " if we specify the timeout then ResourceManager will wait for "
+ "timeout before marking the NodeManager as decommissioned."
+ " The -client|server indicates if the timeout tracking should"
+ " be handled by the client or the ResourceManager. The client"
@@ -234,21 +234,23 @@ public class RMAdminCLI extends HAAdmin {
summary.append("rmadmin is the command to execute YARN administrative " +
"commands.\n");
summary.append("The full syntax is: \n\n" +
- "yarn rmadmin" +
- " [-refreshQueues]" +
- " [-refreshNodes [-g [timeout in seconds] -client|server]]" +
- " [-refreshNodesResources]" +
- " [-refreshSuperUserGroupsConfiguration]" +
- " [-refreshUserToGroupsMappings]" +
- " [-refreshAdminAcls]" +
- " [-refreshServiceAcl]" +
- " [-getGroup [username]]" +
- " [-addToClusterNodeLabels <\"label1(exclusive=true),"
- + "label2(exclusive=false),label3\">]" +
- " [-removeFromClusterNodeLabels <label1,label2,label3>]" +
- " [-replaceLabelsOnNode <\"node1[:port]=label1,label2 node2[:port]=label1\">]" +
- " [-directlyAccessNodeLabelStore]" +
- " [-updateNodeResource [NodeID] [MemSize] [vCores] ([OvercommitTimeout])");
+ "yarn rmadmin" +
+ " [-refreshQueues]" +
+ " [-refreshNodes [-g|graceful [timeout in seconds] -client|server]]" +
+ " [-refreshNodesResources]" +
+ " [-refreshSuperUserGroupsConfiguration]" +
+ " [-refreshUserToGroupsMappings]" +
+ " [-refreshAdminAcls]" +
+ " [-refreshServiceAcl]" +
+ " [-getGroup [username]]" +
+ " [-addToClusterNodeLabels <\"label1(exclusive=true),"
+ + "label2(exclusive=false),label3\">]" +
+ " [-removeFromClusterNodeLabels <label1,label2,label3>]" +
+ " [-replaceLabelsOnNode <\"node1[:port]=label1,label2" +
+ " node2[:port]=label1\">]" +
+ " [-directlyAccessNodeLabelStore]" +
+ " [-updateNodeResource [NodeID] [MemSize] [vCores]" +
+ " ([OvercommitTimeout])");
if (isHAEnabled) {
appendHAUsage(summary);
}
@@ -309,33 +311,40 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
- private int refreshNodes() throws IOException, YarnException {
+ private int refreshNodes(boolean graceful) throws IOException, YarnException {
// Refresh the nodes
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
- RefreshNodesRequest request = RefreshNodesRequest
- .newInstance(DecommissionType.NORMAL);
+ RefreshNodesRequest request = RefreshNodesRequest.newInstance(
+ graceful? DecommissionType.GRACEFUL : DecommissionType.NORMAL);
adminProtocol.refreshNodes(request);
return 0;
}
- private int refreshNodes(long timeout, String trackingMode)
+ private int refreshNodes(int timeout, String trackingMode)
throws IOException, YarnException {
- if (!"client".equals(trackingMode)) {
- throw new UnsupportedOperationException(
- "Only client tracking mode is currently supported.");
- }
+ boolean serverTracking = !"client".equals(trackingMode);
// Graceful decommissioning with timeout
ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
RefreshNodesRequest gracefulRequest = RefreshNodesRequest
- .newInstance(DecommissionType.GRACEFUL);
+ .newInstance(DecommissionType.GRACEFUL, timeout);
adminProtocol.refreshNodes(gracefulRequest);
+ if (serverTracking) {
+ return 0;
+ }
CheckForDecommissioningNodesRequest checkForDecommissioningNodesRequest = recordFactory
.newRecordInstance(CheckForDecommissioningNodesRequest.class);
long waitingTime;
boolean nodesDecommissioning = true;
+ // As RM enforces timeout automatically, client usually don't need
+ // to forcefully decommission nodes upon timeout.
+ // Here we let the client waits a small additional seconds so to avoid
+ // unnecessary double decommission.
+ final int gracePeriod = 5;
// timeout=-1 means wait for all the nodes to be gracefully
// decommissioned
- for (waitingTime = 0; waitingTime < timeout || timeout == -1; waitingTime++) {
+ for (waitingTime = 0;
+ timeout == -1 || (timeout >= 0 && waitingTime < timeout + gracePeriod);
+ waitingTime++) {
// wait for one second to check nodes decommissioning status
try {
Thread.sleep(1000);
@@ -380,6 +389,10 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
+ private int refreshNodes() throws IOException, YarnException {
+ return refreshNodes(false);
+ }
+
private int refreshUserToGroupsMappings() throws IOException,
YarnException {
// Refresh the user-to-groups mappings
@@ -725,33 +738,12 @@ public class RMAdminCLI extends HAAdmin {
return exitCode;
}
}
-
+
try {
if ("-refreshQueues".equals(cmd)) {
exitCode = refreshQueues();
} else if ("-refreshNodes".equals(cmd)) {
- if (args.length == 1) {
- exitCode = refreshNodes();
- } else if (args.length == 3 || args.length == 4) {
- // if the graceful timeout specified
- if ("-g".equals(args[1])) {
- long timeout = -1;
- String trackingMode;
- if (args.length == 4) {
- timeout = validateTimeout(args[2]);
- trackingMode = validateTrackingMode(args[3]);
- } else {
- trackingMode = validateTrackingMode(args[2]);
- }
- exitCode = refreshNodes(timeout, trackingMode);
- } else {
- printUsage(cmd, isHAEnabled);
- return -1;
- }
- } else {
- printUsage(cmd, isHAEnabled);
- return -1;
- }
+ exitCode = handleRefreshNodes(args, cmd, isHAEnabled);
} else if ("-refreshNodesResources".equals(cmd)) {
exitCode = refreshNodesResources();
} else if ("-refreshUserToGroupsMappings".equals(cmd)) {
@@ -768,22 +760,7 @@ public class RMAdminCLI extends HAAdmin {
String[] usernames = Arrays.copyOfRange(args, i, args.length);
exitCode = getGroups(usernames);
} else if ("-updateNodeResource".equals(cmd)) {
- if (args.length < 4 || args.length > 5) {
- System.err.println("Number of parameters specified for " +
- "updateNodeResource is wrong.");
- printUsage(cmd, isHAEnabled);
- exitCode = -1;
- } else {
- String nodeID = args[i++];
- String memSize = args[i++];
- String cores = args[i++];
- int overCommitTimeout = ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT;
- if (i == args.length - 1) {
- overCommitTimeout = Integer.parseInt(args[i]);
- }
- exitCode = updateNodeResource(nodeID, Integer.parseInt(memSize),
- Integer.parseInt(cores), overCommitTimeout);
- }
+ exitCode = handleUpdateNodeResource(args, cmd, isHAEnabled);
} else if ("-addToClusterNodeLabels".equals(cmd)) {
if (i >= args.length) {
System.err.println(NO_LABEL_ERR_MSG);
@@ -843,10 +820,59 @@ public class RMAdminCLI extends HAAdmin {
return exitCode;
}
- private long validateTimeout(String strTimeout) {
- long timeout;
+ // A helper method to reduce the number of lines of run()
+ private int handleRefreshNodes(String[] args, String cmd, boolean isHAEnabled)
+ throws IOException, YarnException {
+ if (args.length == 1) {
+ return refreshNodes();
+ } else if (args.length == 3 || args.length == 4) {
+ // if the graceful timeout specified
+ if ("-g".equals(args[1]) || "-graceful".equals(args[1])) {
+ int timeout = -1;
+ String trackingMode;
+ if (args.length == 4) {
+ timeout = validateTimeout(args[2]);
+ trackingMode = validateTrackingMode(args[3]);
+ } else {
+ trackingMode = validateTrackingMode(args[2]);
+ }
+ return refreshNodes(timeout, trackingMode);
+ } else {
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ }
+ } else {
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ }
+ }
+
+ private int handleUpdateNodeResource(
+ String[] args, String cmd, boolean isHAEnabled)
+ throws NumberFormatException, IOException, YarnException {
+ int i = 1;
+ if (args.length < 4 || args.length > 5) {
+ System.err.println("Number of parameters specified for " +
+ "updateNodeResource is wrong.");
+ printUsage(cmd, isHAEnabled);
+ return -1;
+ } else {
+ String nodeID = args[i++];
+ String memSize = args[i++];
+ String cores = args[i++];
+ int overCommitTimeout = ResourceOption.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT;
+ if (i == args.length - 1) {
+ overCommitTimeout = Integer.parseInt(args[i]);
+ }
+ return updateNodeResource(nodeID, Integer.parseInt(memSize),
+ Integer.parseInt(cores), overCommitTimeout);
+ }
+ }
+
+ private int validateTimeout(String strTimeout) {
+ int timeout;
try {
- timeout = Long.parseLong(strTimeout);
+ timeout = Integer.parseInt(strTimeout);
} catch (NumberFormatException ex) {
throw new IllegalArgumentException(INVALID_TIMEOUT_ERR_MSG + strTimeout);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index d3161ba..60c7eac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -267,7 +267,7 @@ public class TestRMAdminCLI {
CheckForDecommissioningNodesRequest.class))).thenReturn(response);
assertEquals(0, rmAdminCLI.run(args));
verify(admin).refreshNodes(
- RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL));
+ RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL, 1));
verify(admin, never()).refreshNodes(
RefreshNodesRequest.newInstance(DecommissionType.FORCEFUL));
}
@@ -327,7 +327,7 @@ public class TestRMAdminCLI {
});
assertEquals(0, rmAdminCLI.run(args));
verify(admin, atLeastOnce()).refreshNodes(
- RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL));
+ RefreshNodesRequest.newInstance(DecommissionType.GRACEFUL, -1));
verify(admin, never()).refreshNodes(
RefreshNodesRequest.newInstance(DecommissionType.FORCEFUL));
}
@@ -346,10 +346,6 @@ public class TestRMAdminCLI {
String[] negativeTimeoutArgs = {"-refreshNodes", "-g", "-1000", "-client"};
assertEquals(-1, rmAdminCLI.run(negativeTimeoutArgs));
- // server tracking mode
- String[] serveTrackingrArgs = {"-refreshNodes", "-g", "1", "-server"};
- assertEquals(-1, rmAdminCLI.run(serveTrackingrArgs));
-
// invalid tracking mode
String[] invalidTrackingArgs = {"-refreshNodes", "-g", "1", "-foo"};
assertEquals(-1, rmAdminCLI.run(invalidTrackingArgs));
@@ -465,8 +461,9 @@ public class TestRMAdminCLI {
assertTrue(dataOut
.toString()
.contains(
- "yarn rmadmin [-refreshQueues] [-refreshNodes [-g [timeout in " +
- "seconds] -client|server]] [-refreshNodesResources] [-refresh" +
+ "yarn rmadmin [-refreshQueues] [-refreshNodes "+
+ "[-g|graceful [timeout in seconds] -client|server]] " +
+ "[-refreshNodesResources] [-refresh" +
"SuperUserGroupsConfiguration] [-refreshUserToGroupsMappings] " +
"[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup " +
"[username]] [-addToClusterNodeLabels " +
@@ -485,7 +482,8 @@ public class TestRMAdminCLI {
assertTrue(dataOut
.toString()
.contains(
- "-refreshNodes [-g [timeout in seconds] -client|server]: " +
+ "-refreshNodes [-g|graceful [timeout in seconds]" +
+ " -client|server]: " +
"Refresh the hosts information at the ResourceManager."));
assertTrue(dataOut
.toString()
@@ -518,8 +516,8 @@ public class TestRMAdminCLI {
testError(new String[] { "-help", "-refreshQueues" },
"Usage: yarn rmadmin [-refreshQueues]", dataErr, 0);
testError(new String[] { "-help", "-refreshNodes" },
- "Usage: yarn rmadmin [-refreshNodes [-g [timeout in seconds] " +
- "-client|server]]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshNodes [-g|graceful " +
+ "[timeout in seconds] -client|server]]", dataErr, 0);
testError(new String[] { "-help", "-refreshNodesResources" },
"Usage: yarn rmadmin [-refreshNodesResources]", dataErr, 0);
testError(new String[] { "-help", "-refreshUserToGroupsMappings" },
@@ -558,8 +556,8 @@ public class TestRMAdminCLI {
assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
oldOutPrintStream.println(dataOut);
String expectedHelpMsg =
- "yarn rmadmin [-refreshQueues] [-refreshNodes [-g [timeout in "
- + "seconds] -client|server]] "
+ "yarn rmadmin [-refreshQueues] [-refreshNodes [-g|graceful "
+ + "[timeout in seconds] -client|server]] "
+ "[-refreshNodesResources] [-refreshSuperUserGroupsConfiguration] "
+ "[-refreshUserToGroupsMappings] "
+ "[-refreshAdminAcls] [-refreshServiceAcl] [-getGroup"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
index 05f3230..c03a569 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RefreshNodesRequestPBImpl.java
@@ -31,7 +31,6 @@ import com.google.protobuf.TextFormat;
@Private
@Unstable
public class RefreshNodesRequestPBImpl extends RefreshNodesRequest {
-
RefreshNodesRequestProto proto = RefreshNodesRequestProto.getDefaultInstance();
RefreshNodesRequestProto.Builder builder = null;
boolean viaProto = false;
@@ -108,6 +107,22 @@ public class RefreshNodesRequestPBImpl extends RefreshNodesRequest {
return convertFromProtoFormat(p.getDecommissionType());
}
+ @Override
+ public synchronized void setDecommissionTimeout(Integer timeout) {
+ maybeInitBuilder();
+ if (timeout != null) {
+ builder.setDecommissionTimeout(timeout);
+ } else {
+ builder.clearDecommissionTimeout();
+ }
+ }
+
+ @Override
+ public synchronized Integer getDecommissionTimeout() {
+ RefreshNodesRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return p.hasDecommissionTimeout()? p.getDecommissionTimeout() : null;
+ }
+
private DecommissionType convertFromProtoFormat(DecommissionTypeProto p) {
return DecommissionType.valueOf(p.name());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e77e990..e956507 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2586,6 +2586,24 @@
</property>
<property>
+ <description>
+ Timeout in seconds for YARN node graceful decommission.
+ This is the maximal time to wait for running containers and applications to complete
+ before transition a DECOMMISSIONING node into DECOMMISSIONED.
+ </description>
+ <name>yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs</name>
+ <value>3600</value>
+ </property>
+
+ <property>
+ <description>
+ Timeout in seconds of DecommissioningNodesWatcher internal polling.
+ </description>
+ <name>yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs</name>
+ <value>20</value>
+ </property>
+
+ <property>
<description>The Node Label script to run. Script output Line starting with
"NODE_PARTITION:" will be considered as Node Label Partition. In case of
multiple lines have this pattern, then last one will be considered
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 2ec03aa..db55264 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -447,7 +447,8 @@ public class AdminService extends CompositeService implements
rmContext.getNodesListManager().refreshNodes(conf);
break;
case GRACEFUL:
- rmContext.getNodesListManager().refreshNodesGracefully(conf);
+ rmContext.getNodesListManager().refreshNodesGracefully(
+ conf, request.getDecommissionTimeout());
break;
case FORCEFUL:
rmContext.getNodesListManager().refreshNodesForcefully();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
new file mode 100644
index 0000000..376b503
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
@@ -0,0 +1,439 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+/**
+ * DecommissioningNodesWatcher is used by ResourceTrackerService to track
+ * DECOMMISSIONING nodes to decide when, after all running containers on
+ * the node have completed, will be transitioned into DECOMMISSIONED state
+ * (NodeManager will be told to shutdown).
+ * Under MR application, a node, after completes all its containers,
+ * may still serve it map output data during the duration of the application
+ * for reducers. A fully graceful mechanism would keep such DECOMMISSIONING
+ * nodes until all involved applications complete. It could be however
+ * undesirable under long-running applications scenario where a bunch
+ * of "idle" nodes would stay around for long period of time.
+ *
+ * DecommissioningNodesWatcher balance such concern with a timeout policy ---
+ * a DECOMMISSIONING node will be DECOMMISSIONED no later than
+ * DECOMMISSIONING_TIMEOUT regardless of running containers or applications.
+ *
+ * To be efficient, DecommissioningNodesWatcher skip tracking application
+ * containers on a particular node before the node is in DECOMMISSIONING state.
+ * It only tracks containers once the node is in DECOMMISSIONING state.
+ * DecommissioningNodesWatcher basically is no cost when no node is
+ * DECOMMISSIONING. This sacrifices the possibility that the node once
+ * host containers of an application that is still running
+ * (the affected map tasks will be rescheduled).
+ */
+public class DecommissioningNodesWatcher {
+ private static final Log LOG =
+ LogFactory.getLog(DecommissioningNodesWatcher.class);
+
+ private final RMContext rmContext;
+
+ // Default timeout value in mills.
+ // Negative value indicates no timeout. 0 means immediate.
+ private long defaultTimeoutMs =
+ 1000L * YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT;
+
+ // Once a RMNode is observed in DECOMMISSIONING state,
+ // All its ContainerStatus update are tracked inside DecomNodeContext.
+ class DecommissioningNodeContext {
+ private final NodeId nodeId;
+
+ // Last known NodeState.
+ private NodeState nodeState;
+
+ // The moment node is observed in DECOMMISSIONING state.
+ private final long decommissioningStartTime;
+
+ private long lastContainerFinishTime;
+
+ // number of running containers at the moment.
+ private int numActiveContainers;
+
+ // All applications run on the node at or after decommissioningStartTime.
+ private Set<ApplicationId> appIds;
+
+ // First moment the node is observed in DECOMMISSIONED state.
+ private long decommissionedTime;
+
+ // Timeout in millis for this decommissioning node.
+ // This value could be dynamically updated with new value from RMNode.
+ private long timeoutMs;
+
+ private long lastUpdateTime;
+
+ public DecommissioningNodeContext(NodeId nodeId) {
+ this.nodeId = nodeId;
+ this.appIds = new HashSet<ApplicationId>();
+ this.decommissioningStartTime = mclock.getTime();
+ this.timeoutMs = defaultTimeoutMs;
+ }
+
+ void updateTimeout(Integer timeoutSec) {
+ this.timeoutMs = (timeoutSec == null)?
+ defaultTimeoutMs : (1000L * timeoutSec);
+ }
+ }
+
+ // All DECOMMISSIONING nodes to track.
+ private HashMap<NodeId, DecommissioningNodeContext> decomNodes =
+ new HashMap<NodeId, DecommissioningNodeContext>();
+
+ private Timer pollTimer;
+ private MonotonicClock mclock;
+
+ public DecommissioningNodesWatcher(RMContext rmContext) {
+ this.rmContext = rmContext;
+ pollTimer = new Timer(true);
+ mclock = new MonotonicClock();
+ }
+
+ public void init(Configuration conf) {
+ readDecommissioningTimeout(conf);
+ int v = conf.getInt(
+ YarnConfiguration.RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL,
+ YarnConfiguration
+ .DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL);
+ pollTimer.schedule(new PollTimerTask(rmContext), 0, (1000L * v));
+ }
+
+ /**
+ * Update rmNode decommissioning status based on NodeStatus.
+ * @param rmNode The node
+ * @param remoteNodeStatus latest NodeStatus
+ */
+ public synchronized void update(RMNode rmNode, NodeStatus remoteNodeStatus) {
+ DecommissioningNodeContext context = decomNodes.get(rmNode.getNodeID());
+ long now = mclock.getTime();
+ if (rmNode.getState() == NodeState.DECOMMISSIONED) {
+ if (context == null) {
+ return;
+ }
+ context.nodeState = rmNode.getState();
+ // keep DECOMMISSIONED node for a while for status log, so that such
+ // host will appear as DECOMMISSIONED instead of quietly disappears.
+ if (context.decommissionedTime == 0) {
+ context.decommissionedTime = now;
+ } else if (now - context.decommissionedTime > 60000L) {
+ decomNodes.remove(rmNode.getNodeID());
+ }
+ } else if (rmNode.getState() == NodeState.DECOMMISSIONING) {
+ if (context == null) {
+ context = new DecommissioningNodeContext(rmNode.getNodeID());
+ decomNodes.put(rmNode.getNodeID(), context);
+ context.nodeState = rmNode.getState();
+ context.decommissionedTime = 0;
+ }
+ context.updateTimeout(rmNode.getDecommissioningTimeout());
+ context.lastUpdateTime = now;
+
+ if (remoteNodeStatus.getKeepAliveApplications() != null) {
+ context.appIds.addAll(remoteNodeStatus.getKeepAliveApplications());
+ }
+
+ // Count number of active containers.
+ int numActiveContainers = 0;
+ for (ContainerStatus cs : remoteNodeStatus.getContainersStatuses()) {
+ ContainerState newState = cs.getState();
+ if (newState == ContainerState.RUNNING ||
+ newState == ContainerState.NEW) {
+ numActiveContainers++;
+ }
+ context.numActiveContainers = numActiveContainers;
+ ApplicationId aid = cs.getContainerId()
+ .getApplicationAttemptId().getApplicationId();
+ if (!context.appIds.contains(aid)) {
+ context.appIds.add(aid);
+ }
+ }
+
+ context.numActiveContainers = numActiveContainers;
+
+ // maintain lastContainerFinishTime.
+ if (context.numActiveContainers == 0 &&
+ context.lastContainerFinishTime == 0) {
+ context.lastContainerFinishTime = now;
+ }
+ } else {
+ // remove node in other states
+ if (context != null) {
+ decomNodes.remove(rmNode.getNodeID());
+ }
+ }
+ }
+
+ public synchronized void remove(NodeId nodeId) {
+ DecommissioningNodeContext context = decomNodes.get(nodeId);
+ if (context != null) {
+ LOG.info("remove " + nodeId + " in " + context.nodeState);
+ decomNodes.remove(nodeId);
+ }
+ }
+
+ /**
+ * Status about a specific decommissioning node.
+ *
+ */
+ public enum DecommissioningNodeStatus {
+ // Node is not in DECOMMISSIONING state.
+ NONE,
+
+ // wait for running containers to complete
+ WAIT_CONTAINER,
+
+ // wait for running application to complete (after all containers complete);
+ WAIT_APP,
+
+ // Timeout waiting for either containers or applications to complete.
+ TIMEOUT,
+
+ // nothing to wait, ready to be decommissioned
+ READY,
+
+ // The node has already been decommissioned
+ DECOMMISSIONED,
+ }
+
+ public boolean checkReadyToBeDecommissioned(NodeId nodeId) {
+ DecommissioningNodeStatus s = checkDecommissioningStatus(nodeId);
+ return (s == DecommissioningNodeStatus.READY ||
+ s == DecommissioningNodeStatus.TIMEOUT);
+ }
+
+ public DecommissioningNodeStatus checkDecommissioningStatus(NodeId nodeId) {
+ DecommissioningNodeContext context = decomNodes.get(nodeId);
+ if (context == null) {
+ return DecommissioningNodeStatus.NONE;
+ }
+
+ if (context.nodeState == NodeState.DECOMMISSIONED) {
+ return DecommissioningNodeStatus.DECOMMISSIONED;
+ }
+
+ long waitTime = mclock.getTime() - context.decommissioningStartTime;
+ if (context.numActiveContainers > 0) {
+ return (context.timeoutMs < 0 || waitTime < context.timeoutMs)?
+ DecommissioningNodeStatus.WAIT_CONTAINER :
+ DecommissioningNodeStatus.TIMEOUT;
+ }
+
+ removeCompletedApps(context);
+ if (context.appIds.size() == 0) {
+ return DecommissioningNodeStatus.READY;
+ } else {
+ return (context.timeoutMs < 0 || waitTime < context.timeoutMs)?
+ DecommissioningNodeStatus.WAIT_APP :
+ DecommissioningNodeStatus.TIMEOUT;
+ }
+ }
+
+ /**
+ * PollTimerTask periodically:
+ * 1. log status of all DECOMMISSIONING nodes;
+ * 2. identify and taken care of stale DECOMMISSIONING nodes
+ * (for example, node already terminated).
+ */
+ class PollTimerTask extends TimerTask {
+ private final RMContext rmContext;
+
+ public PollTimerTask(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ public void run() {
+ logDecommissioningNodesStatus();
+ long now = mclock.getTime();
+ Set<NodeId> staleNodes = new HashSet<NodeId>();
+
+ for (Iterator<Map.Entry<NodeId, DecommissioningNodeContext>> it =
+ decomNodes.entrySet().iterator(); it.hasNext();) {
+ Map.Entry<NodeId, DecommissioningNodeContext> e = it.next();
+ DecommissioningNodeContext d = e.getValue();
+ // Skip node recently updated (NM usually updates every second).
+ if (now - d.lastUpdateTime < 5000L) {
+ continue;
+ }
+ // Remove stale non-DECOMMISSIONING node
+ if (d.nodeState != NodeState.DECOMMISSIONING) {
+ LOG.debug("remove " + d.nodeState + " " + d.nodeId);
+ it.remove();
+ continue;
+ } else if (now - d.lastUpdateTime > 60000L) {
+ // Node DECOMMISSIONED could become stale, remove as necessary.
+ RMNode rmNode = getRmNode(d.nodeId);
+ if (rmNode != null &&
+ rmNode.getState() == NodeState.DECOMMISSIONED) {
+ LOG.debug("remove " + rmNode.getState() + " " + d.nodeId);
+ it.remove();
+ continue;
+ }
+ }
+ if (d.timeoutMs >= 0 &&
+ d.decommissioningStartTime + d.timeoutMs < now) {
+ staleNodes.add(d.nodeId);
+ LOG.debug("Identified stale and timeout node " + d.nodeId);
+ }
+ }
+
+ for (NodeId nodeId : staleNodes) {
+ RMNode rmNode = this.rmContext.getRMNodes().get(nodeId);
+ if (rmNode == null || rmNode.getState() != NodeState.DECOMMISSIONING) {
+ remove(nodeId);
+ continue;
+ }
+ if (rmNode.getState() == NodeState.DECOMMISSIONING &&
+ checkReadyToBeDecommissioned(rmNode.getNodeID())) {
+ LOG.info("DECOMMISSIONING " + nodeId + " timeout");
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
+ }
+ }
+ }
+ }
+
+ private RMNode getRmNode(NodeId nodeId) {
+ RMNode rmNode = this.rmContext.getRMNodes().get(nodeId);
+ if (rmNode == null) {
+ rmNode = this.rmContext.getInactiveRMNodes().get(nodeId);
+ }
+ return rmNode;
+ }
+
+ private void removeCompletedApps(DecommissioningNodeContext context) {
+ Iterator<ApplicationId> it = context.appIds.iterator();
+ while (it.hasNext()) {
+ ApplicationId appId = it.next();
+ RMApp rmApp = rmContext.getRMApps().get(appId);
+ if (rmApp == null) {
+ LOG.debug("Consider non-existing app " + appId + " as completed");
+ it.remove();
+ continue;
+ }
+ if (rmApp.getState() == RMAppState.FINISHED ||
+ rmApp.getState() == RMAppState.FAILED ||
+ rmApp.getState() == RMAppState.KILLED) {
+ LOG.debug("Remove " + rmApp.getState() + " app " + appId);
+ it.remove();
+ }
+ }
+ }
+
+ // Time in second to be decommissioned.
+ private int getTimeoutInSec(DecommissioningNodeContext context) {
+ if (context.nodeState == NodeState.DECOMMISSIONED) {
+ return 0;
+ } else if (context.nodeState != NodeState.DECOMMISSIONING) {
+ return -1;
+ }
+ if (context.appIds.size() == 0 && context.numActiveContainers == 0) {
+ return 0;
+ }
+ // negative timeout value means no timeout (infinite timeout).
+ if (context.timeoutMs < 0) {
+ return -1;
+ }
+
+ long now = mclock.getTime();
+ long timeout = context.decommissioningStartTime + context.timeoutMs - now;
+ return Math.max(0, (int)(timeout / 1000));
+ }
+
+ private void logDecommissioningNodesStatus() {
+ if (!LOG.isDebugEnabled() || decomNodes.size() == 0) {
+ return;
+ }
+ StringBuilder sb = new StringBuilder();
+ long now = mclock.getTime();
+ for (DecommissioningNodeContext d : decomNodes.values()) {
+ DecommissioningNodeStatus s = checkDecommissioningStatus(d.nodeId);
+ sb.append(String.format(
+ "%n %-34s %4ds fresh:%3ds containers:%2d %14s",
+ d.nodeId.getHost(),
+ (now - d.decommissioningStartTime) / 1000,
+ (now - d.lastUpdateTime) / 1000,
+ d.numActiveContainers,
+ s));
+ if (s == DecommissioningNodeStatus.WAIT_APP ||
+ s == DecommissioningNodeStatus.WAIT_CONTAINER) {
+ sb.append(String.format(" timeout:%4ds", getTimeoutInSec(d)));
+ }
+ for (ApplicationId aid : d.appIds) {
+ sb.append("\n " + aid);
+ RMApp rmApp = rmContext.getRMApps().get(aid);
+ if (rmApp != null) {
+ sb.append(String.format(
+ " %s %9s %5.2f%% %5ds",
+ rmApp.getState(),
+ (rmApp.getApplicationType() == null)?
+ "" : rmApp.getApplicationType(),
+ 100.0 * rmApp.getProgress(),
+ (mclock.getTime() - rmApp.getStartTime()) / 1000));
+ }
+ }
+ }
+ LOG.info("Decommissioning Nodes: " + sb.toString());
+ }
+
+ // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.
+ // This enables DecommissioningNodesWatcher to pick up new value
+ // without ResourceManager restart.
+ private void readDecommissioningTimeout(Configuration conf) {
+ try {
+ if (conf == null) {
+ conf = new YarnConfiguration();
+ }
+ int v = conf.getInt(
+ YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
+ if (defaultTimeoutMs != 1000L * v) {
+ defaultTimeoutMs = 1000L * v;
+ LOG.info("Use new decommissioningTimeoutMs: " + defaultTimeoutMs);
+ }
+ } catch (Exception e) {
+ LOG.info("Error readDecommissioningTimeout ", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index 7937383..99413bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -19,14 +19,18 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.Map;
-import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,6 +41,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -47,14 +52,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecommissioningEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
-
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import com.google.common.annotations.VisibleForTesting;
+
@SuppressWarnings("unchecked")
public class NodesListManager extends CompositeService implements
EventHandler<NodesListManagerEvent> {
@@ -178,10 +184,11 @@ public class NodesListManager extends CompositeService implements
if (!LOG.isDebugEnabled()) {
return;
}
-
- LOG.debug("hostsReader: in=" + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
+
+ LOG.debug("hostsReader: in=" +
+ conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH) + " out=" +
- conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
+ conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH));
Set<String> hostsList = new HashSet<String>();
@@ -196,23 +203,19 @@ public class NodesListManager extends CompositeService implements
}
}
- public void refreshNodes(Configuration yarnConf) throws IOException,
- YarnException {
- refreshHostsReader(yarnConf);
+ public void refreshNodes(Configuration yarnConf)
+ throws IOException, YarnException {
+ refreshNodes(yarnConf, false);
+ }
- for (NodeId nodeId: rmContext.getRMNodes().keySet()) {
- if (!isValidNode(nodeId.getHost())) {
- RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
- RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, nodeEventType));
- }
- }
- updateInactiveNodes();
+ public void refreshNodes(Configuration yarnConf, boolean graceful)
+ throws IOException, YarnException {
+ refreshHostsReader(yarnConf, graceful, null);
}
- private void refreshHostsReader(Configuration yarnConf) throws IOException,
- YarnException {
+ private void refreshHostsReader(
+ Configuration yarnConf, boolean graceful, Integer timeout)
+ throws IOException, YarnException {
if (null == yarnConf) {
yarnConf = new YarnConfiguration();
}
@@ -222,8 +225,16 @@ public class NodesListManager extends CompositeService implements
excludesFile =
yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
+ LOG.info("refreshNodes excludesFile " + excludesFile);
hostsReader.refresh(includesFile, excludesFile);
printConfiguredHosts();
+
+ LOG.info("hostsReader include:{" +
+ StringUtils.join(",", hostsReader.getHosts()) +
+ "} exclude:{" +
+ StringUtils.join(",", hostsReader.getExcludedHosts()) + "}");
+
+ handleExcludeNodeList(graceful, timeout);
}
private void setDecomissionedNMs() {
@@ -237,6 +248,86 @@ public class NodesListManager extends CompositeService implements
}
}
+ // Handle excluded nodes based on following rules:
+ // Recommission DECOMMISSIONED or DECOMMISSIONING nodes no longer excluded;
+ // Gracefully decommission excluded nodes that are not already
+ // DECOMMISSIONED nor DECOMMISSIONING; Take no action for excluded nodes
+ // that are already DECOMMISSIONED or DECOMMISSIONING.
+ private void handleExcludeNodeList(boolean graceful, Integer timeout) {
+ // DECOMMISSIONED/DECOMMISSIONING nodes need to be re-commissioned.
+ List<RMNode> nodesToRecom = new ArrayList<RMNode>();
+
+ // Nodes need to be decommissioned (graceful or forceful);
+ List<RMNode> nodesToDecom = new ArrayList<RMNode>();
+
+ Set<String> includes = new HashSet<String>();
+ Map<String, Integer> excludes = new HashMap<String, Integer>();
+ hostsReader.getHostDetails(includes, excludes);
+
+ for (RMNode n : this.rmContext.getRMNodes().values()) {
+ NodeState s = n.getState();
+ // An invalid node (either due to explicit exclude or not include)
+ // should be excluded.
+ boolean isExcluded = !isValidNode(
+ n.getHostName(), includes, excludes.keySet());
+ String nodeStr = "node " + n.getNodeID() + " with state " + s;
+ if (!isExcluded) {
+ // Note that no action is needed for DECOMMISSIONED node.
+ if (s == NodeState.DECOMMISSIONING) {
+ LOG.info("Recommission " + nodeStr);
+ nodesToRecom.add(n);
+ }
+ // Otherwise no-action needed.
+ } else {
+ // exclude is true.
+ if (graceful) {
+ // Use per node timeout if exist otherwise the request timeout.
+ Integer timeoutToUse = (excludes.get(n.getHostName()) != null)?
+ excludes.get(n.getHostName()) : timeout;
+ if (s != NodeState.DECOMMISSIONED &&
+ s != NodeState.DECOMMISSIONING) {
+ LOG.info("Gracefully decommission " + nodeStr);
+ nodesToDecom.add(n);
+ } else if (s == NodeState.DECOMMISSIONING &&
+ !Objects.equals(n.getDecommissioningTimeout(),
+ timeoutToUse)) {
+ LOG.info("Update " + nodeStr + " timeout to be " + timeoutToUse);
+ nodesToDecom.add(n);
+ } else {
+ LOG.info("No action for " + nodeStr);
+ }
+ } else {
+ if (s != NodeState.DECOMMISSIONED) {
+ LOG.info("Forcefully decommission " + nodeStr);
+ nodesToDecom.add(n);
+ }
+ }
+ }
+ }
+
+ for (RMNode n : nodesToRecom) {
+ RMNodeEvent e = new RMNodeEvent(
+ n.getNodeID(), RMNodeEventType.RECOMMISSION);
+ this.rmContext.getDispatcher().getEventHandler().handle(e);
+ }
+
+ for (RMNode n : nodesToDecom) {
+ RMNodeEvent e;
+ if (graceful) {
+ Integer timeoutToUse = (excludes.get(n.getHostName()) != null)?
+ excludes.get(n.getHostName()) : timeout;
+ e = new RMNodeDecommissioningEvent(n.getNodeID(), timeoutToUse);
+ } else {
+ RMNodeEventType eventType = isUntrackedNode(n.getHostName())?
+ RMNodeEventType.SHUTDOWN : RMNodeEventType.DECOMMISSION;
+ e = new RMNodeEvent(n.getNodeID(), eventType);
+ }
+ this.rmContext.getDispatcher().getEventHandler().handle(e);
+ }
+
+ updateInactiveNodes();
+ }
+
@VisibleForTesting
public int getNodeRemovalCheckInterval() {
return nodeRemovalCheckInterval;
@@ -360,11 +451,15 @@ public class NodesListManager extends CompositeService implements
}
public boolean isValidNode(String hostName) {
- String ip = resolver.resolve(hostName);
Set<String> hostsList = new HashSet<String>();
Set<String> excludeList = new HashSet<String>();
hostsReader.getHostDetails(hostsList, excludeList);
+ return isValidNode(hostName, hostsList, excludeList);
+ }
+ private boolean isValidNode(
+ String hostName, Set<String> hostsList, Set<String> excludeList) {
+ String ip = resolver.resolve(hostName);
return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList
.contains(ip))
&& !(excludeList.contains(hostName) || excludeList.contains(ip));
@@ -478,29 +573,14 @@ public class NodesListManager extends CompositeService implements
/**
* Refresh the nodes gracefully
*
- * @param conf
+ * @param yarnConf
+ * @param timeout decommission timeout, null means default timeout.
* @throws IOException
* @throws YarnException
*/
- public void refreshNodesGracefully(Configuration conf) throws IOException,
- YarnException {
- refreshHostsReader(conf);
- for (Entry<NodeId, RMNode> entry : rmContext.getRMNodes().entrySet()) {
- NodeId nodeId = entry.getKey();
- if (!isValidNode(nodeId.getHost())) {
- RMNodeEventType nodeEventType = isUntrackedNode(nodeId.getHost()) ?
- RMNodeEventType.SHUTDOWN : RMNodeEventType.GRACEFUL_DECOMMISSION;
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeEvent(nodeId, nodeEventType));
- } else {
- // Recommissioning the nodes
- if (entry.getValue().getState() == NodeState.DECOMMISSIONING) {
- this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMNodeEvent(nodeId, RMNodeEventType.RECOMMISSION));
- }
- }
- }
- updateInactiveNodes();
+ public void refreshNodesGracefully(Configuration yarnConf, Integer timeout)
+ throws IOException, YarnException {
+ refreshHostsReader(yarnConf, true, timeout);
}
/**
@@ -596,4 +676,4 @@ public class NodesListManager extends CompositeService implements
this.host = hst;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 9b9b02e..5e9827a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -70,7 +70,7 @@ public class RMServerUtils {
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
- // nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
+ // nodes contains nodes that are NEW, RUNNING, UNHEALTHY or DECOMMISSIONING.
ArrayList<RMNode> results = new ArrayList<RMNode>();
if (acceptedStates.contains(NodeState.NEW) ||
acceptedStates.contains(NodeState.RUNNING) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 9d480f3..51fc0bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -113,6 +113,8 @@ public class ResourceTrackerService extends AbstractService implements
private int minAllocMb;
private int minAllocVcores;
+ private DecommissioningNodesWatcher decommissioningWatcher;
+
private boolean isDistributedNodeLabelsConf;
private boolean isDelegatedCentralizedNodeLabelsConf;
private DynamicResourceConfiguration drConf;
@@ -131,6 +133,7 @@ public class ResourceTrackerService extends AbstractService implements
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+ this.decommissioningWatcher = new DecommissioningNodesWatcher(rmContext);
}
@Override
@@ -170,6 +173,7 @@ public class ResourceTrackerService extends AbstractService implements
}
loadDynamicResourceConfiguration(conf);
+ decommissioningWatcher.init(conf);
super.serviceInit(conf);
}
@@ -494,6 +498,7 @@ public class ResourceTrackerService extends AbstractService implements
// Send ping
this.nmLivelinessMonitor.receivedPing(nodeId);
+ this.decommissioningWatcher.update(rmNode, remoteNodeStatus);
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse();
@@ -526,6 +531,20 @@ public class ResourceTrackerService extends AbstractService implements
updateAppCollectorsMap(request);
}
+ // Evaluate whether a DECOMMISSIONING node is ready to be DECOMMISSIONED.
+ if (rmNode.getState() == NodeState.DECOMMISSIONING &&
+ decommissioningWatcher.checkReadyToBeDecommissioned(
+ rmNode.getNodeID())) {
+ String message = "DECOMMISSIONING " + nodeId +
+ " is ready to be decommissioned";
+ LOG.info(message);
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
+ this.nmLivelinessMonitor.unregister(nodeId);
+ return YarnServerBuilderUtils.newNodeHeartbeatResponse(
+ NodeAction.SHUTDOWN, message);
+ }
+
// Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
.newNodeHeartbeatResponse(lastNodeHeartbeatResponse.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 3a9cf54..10e2afa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -175,4 +175,10 @@ public interface RMNode {
long getUntrackedTimeStamp();
void setUntrackedTimeStamp(long timeStamp);
+ /*
+ * Optional decommissioning timeout in second
+ * (null indicates default timeout).
+ * @return the decommissioning timeout in second.
+ */
+ Integer getDecommissioningTimeout();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
new file mode 100644
index 0000000..9955e9e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecommissioningEvent.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+/**
+ * RMNode Decommissioning Event.
+ *
+ */
+public class RMNodeDecommissioningEvent extends RMNodeEvent {
+ // Optional decommissioning timeout in second.
+ private final Integer decommissioningTimeout;
+
+ // Create instance with optional timeout
+ // (timeout could be null which means use default).
+ public RMNodeDecommissioningEvent(NodeId nodeId, Integer timeout) {
+ super(nodeId, RMNodeEventType.GRACEFUL_DECOMMISSION);
+ this.decommissioningTimeout = timeout;
+ }
+
+ public Integer getDecommissioningTimeout() {
+ return this.decommissioningTimeout;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index a3a6b30..d1ccecb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -124,6 +125,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private String healthReport;
private long lastHealthReportTime;
private String nodeManagerVersion;
+ private Integer decommissioningTimeout;
private long timeStamp;
/* Aggregated resource utilization for the containers. */
@@ -179,7 +181,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
NodeState,
RMNodeEventType,
RMNodeEvent>(NodeState.NEW)
-
//Transitions from NEW state
.addTransition(NodeState.NEW, NodeState.RUNNING,
RMNodeEventType.STARTED, new AddNodeTransition())
@@ -265,6 +266,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
.addTransition(NodeState.DECOMMISSIONING, NodeState.REBOOTED,
RMNodeEventType.REBOOTING,
new DeactivateNodeTransition(NodeState.REBOOTED))
+ .addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
+ RMNodeEventType.FINISHED_CONTAINERS_PULLED_BY_AM,
+ new AddContainersToBeRemovedFromNMTransition())
.addTransition(NodeState.DECOMMISSIONING, NodeState.DECOMMISSIONING,
RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition())
@@ -633,7 +637,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
} catch (InvalidStateTransitionException e) {
LOG.error("Can't handle this event at current state", e);
LOG.error("Invalid event " + event.getType() +
- " on Node " + this.nodeId);
+ " on Node " + this.nodeId + " oldState " + oldState);
}
if (oldState != getState()) {
LOG.info(nodeId + " Node Transitioned from " + oldState + " to "
@@ -666,6 +670,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
case SHUTDOWN:
metrics.decrNumShutdownNMs();
break;
+ case DECOMMISSIONING:
+ metrics.decrDecommissioningNMs();
+ break;
default:
LOG.debug("Unexpected previous node state");
}
@@ -712,6 +719,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
case DECOMMISSIONING:
metrics.decrDecommissioningNMs();
break;
+ case DECOMMISSIONED:
+ metrics.decrDecommisionedNMs();
+ break;
case UNHEALTHY:
metrics.decrNumUnhealthyNMs();
break;
@@ -1087,9 +1097,26 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ Integer timeout = null;
+ if (RMNodeDecommissioningEvent.class.isInstance(event)) {
+ RMNodeDecommissioningEvent e = ((RMNodeDecommissioningEvent) event);
+ timeout = e.getDecommissioningTimeout();
+ }
+ // Pick up possible updates on decommissioningTimeout.
+ if (rmNode.getState() == NodeState.DECOMMISSIONING) {
+ if (!Objects.equals(rmNode.getDecommissioningTimeout(), timeout)) {
+ LOG.info("Update " + rmNode.getNodeID() +
+ " DecommissioningTimeout to be " + timeout);
+ rmNode.decommissioningTimeout = timeout;
+ } else {
+ LOG.info(rmNode.getNodeID() + " is already DECOMMISSIONING");
+ }
+ return;
+ }
LOG.info("Put Node " + rmNode.nodeId + " in DECOMMISSIONING.");
// Update NM metrics during graceful decommissioning.
rmNode.updateMetricsForGracefulDecommission(initState, finalState);
+ rmNode.decommissioningTimeout = timeout;
if (rmNode.originalTotalCapability == null){
rmNode.originalTotalCapability =
Resources.clone(rmNode.totalCapability);
@@ -1156,24 +1183,6 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return NodeState.UNHEALTHY;
}
}
- if (isNodeDecommissioning) {
- List<ApplicationId> runningApps = rmNode.getRunningApps();
-
- List<ApplicationId> keepAliveApps = statusEvent.getKeepAliveAppIds();
-
- // no running (and keeping alive) app on this node, get it
- // decommissioned.
- // TODO may need to check no container is being scheduled on this node
- // as well.
- if ((runningApps == null || runningApps.size() == 0)
- && (keepAliveApps == null || keepAliveApps.size() == 0)) {
- RMNodeImpl.deactivateNode(rmNode, NodeState.DECOMMISSIONED);
- return NodeState.DECOMMISSIONED;
- }
-
- // TODO (in YARN-3223) if node in decommissioning, get node resource
- // updated if container get finished (keep available resource to be 0)
- }
rmNode.handleContainerStatus(statusEvent.getContainers());
rmNode.handleReportedIncreasedContainers(
@@ -1472,4 +1481,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
public void setUntrackedTimeStamp(long ts) {
this.timeStamp = ts;
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ return decommissioningTimeout;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0da69c32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
index 3012d0d..1789e09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
@@ -97,7 +97,7 @@ public class ClusterMetricsInfo {
this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
this.shutdownNodes = clusterMetrics.getNumShutdownNMs();
this.totalNodes = activeNodes + lostNodes + decommissionedNodes
- + rebootedNodes + unhealthyNodes + shutdownNodes;
+ + rebootedNodes + unhealthyNodes + decommissioningNodes + shutdownNodes;
}
public int getAppsSubmitted() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-10744. Internally optimize
path component resolution. Contributed by Daryn Sharp.
Posted by cn...@apache.org.
HDFS-10744. Internally optimize path component resolution. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03dea65e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03dea65e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03dea65e
Branch: refs/heads/HADOOP-13345
Commit: 03dea65e0b17ca2f9460bb6110f6ab3a321b8bf2
Parents: d714030
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Aug 15 16:44:18 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Aug 15 16:45:44 2016 -0500
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 18 ++--
.../hdfs/server/namenode/FSDirAppendOp.java | 4 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 22 ++--
.../hdfs/server/namenode/FSDirDeleteOp.java | 5 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 8 +-
.../server/namenode/FSDirErasureCodingOp.java | 8 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 3 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 12 +--
.../server/namenode/FSDirStatAndListingOp.java | 27 ++---
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 4 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 15 +--
.../hdfs/server/namenode/FSDirXAttrOp.java | 13 +--
.../hdfs/server/namenode/FSDirectory.java | 100 +++++++++----------
.../hdfs/server/namenode/FSNamesystem.java | 16 +--
.../hdfs/server/namenode/TestINodeFile.java | 53 +++++-----
16 files changed, 118 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index 0c572b5..296bed2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -39,8 +39,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@@ -65,8 +64,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@@ -90,8 +88,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@@ -115,8 +112,7 @@ class FSDirAclOp {
String src = srcArg;
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@@ -135,9 +131,8 @@ class FSDirAclOp {
throws IOException {
String src = srcArg;
checkAclsConfigFlag(fsd);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip;
fsd.writeLock();
try {
@@ -155,8 +150,7 @@ class FSDirAclOp {
FSDirectory fsd, String src) throws IOException {
checkAclsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
String srcs = FSDirectory.normalizePath(src);
fsd.readLock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 88d706b..daf5c71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -82,14 +82,12 @@ final class FSDirAppendOp {
final boolean logRetryCache) throws IOException {
assert fsn.hasWriteLock();
- final byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(srcArg);
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
final String src;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
final INodesInPath iip = fsd.getINodesInPath4Write(src);
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 06942f8..3dc1c30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -57,11 +57,10 @@ public class FSDirAttrOp {
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
unprotectedSetPermission(fsd, src, permission);
@@ -79,11 +78,10 @@ public class FSDirAttrOp {
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
if (!pc.isSuperUser()) {
@@ -106,12 +104,11 @@ public class FSDirAttrOp {
FSDirectory fsd, String src, long mtime, long atime)
throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
// Write access is required to set access and modification times
if (fsd.isPermissionEnabled()) {
@@ -139,10 +136,9 @@ public class FSDirAttrOp {
bm.verifyReplication(src, replication, null);
final boolean isFile;
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
@@ -187,11 +183,10 @@ public class FSDirAttrOp {
DFS_STORAGE_POLICY_ENABLED_KEY));
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.writeLock();
try {
- src = FSDirectory.resolvePath(src, pathComponents, fsd);
+ src = FSDirectory.resolvePath(src, fsd);
iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
@@ -214,11 +209,9 @@ public class FSDirAttrOp {
static BlockStoragePolicy getStoragePolicy(FSDirectory fsd, BlockManager bm,
String path) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(path);
fsd.readLock();
try {
- path = fsd.resolvePath(pc, path, pathComponents);
+ path = fsd.resolvePath(pc, path);
final INodesInPath iip = fsd.getINodesInPath(path, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
@@ -237,10 +230,9 @@ public class FSDirAttrOp {
static long getPreferredBlockSize(FSDirectory fsd, String src)
throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
index 6db2ce8..58f7a61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
@@ -97,9 +97,8 @@ class FSDirDeleteOp {
throws IOException {
FSDirectory fsd = fsn.getFSDirectory();
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath4Write(src, false);
if (!recursive && fsd.isNonEmptyDirectory(iip)) {
throw new PathIsNotEmptyDirectoryException(src + " is non empty");
@@ -109,7 +108,7 @@ class FSDirDeleteOp {
FsAction.ALL, true);
}
if (recursive && fsd.isNonEmptyDirectory(iip)) {
- checkProtectedDescendants(fsd, fsd.normalizePath(src));
+ checkProtectedDescendants(fsd, src);
}
return deleteInternal(fsn, src, iip, logRetryCache);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index f3e4d46..ba9e9d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -146,8 +146,6 @@ final class FSDirEncryptionZoneOp {
static HdfsFileStatus createEncryptionZone(final FSDirectory fsd,
final String srcArg, final FSPermissionChecker pc, final String cipher,
final String keyName, final boolean logRetryCache) throws IOException {
- final byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(srcArg);
final CipherSuite suite = CipherSuite.convert(cipher);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
final String src;
@@ -157,7 +155,7 @@ final class FSDirEncryptionZoneOp {
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
final XAttr ezXAttr = fsd.ezManager.createEncryptionZone(src, suite,
version, keyName);
xAttrs.add(ezXAttr);
@@ -180,14 +178,12 @@ final class FSDirEncryptionZoneOp {
static Map.Entry<EncryptionZone, HdfsFileStatus> getEZForPath(
final FSDirectory fsd, final String srcArg, final FSPermissionChecker pc)
throws IOException {
- final byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(srcArg);
final String src;
final INodesInPath iip;
final EncryptionZone ret;
fsd.readLock();
try {
- src = fsd.resolvePath(pc, srcArg, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
iip = fsd.getINodesInPath(src, true);
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 7405e1f..d46342b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -71,11 +71,9 @@ final class FSDirErasureCodingOp {
String src = srcArg;
FSPermissionChecker pc = null;
- byte[][] pathComponents = null;
- pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
pc = fsn.getPermissionChecker();
FSDirectory fsd = fsn.getFSDirectory();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip;
List<XAttr> xAttrs;
fsd.writeLock();
@@ -224,11 +222,9 @@ final class FSDirErasureCodingOp {
private static INodesInPath getINodesInPath(final FSNamesystem fsn,
final String srcArg) throws IOException {
String src = srcArg;
- final byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(src);
final FSDirectory fsd = fsn.getFSDirectory();
final FSPermissionChecker pc = fsn.getPermissionChecker();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsn.isPermissionEnabled()) {
fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index c74faca..1141422 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -50,10 +50,9 @@ class FSDirMkdirOp {
throw new InvalidPathException(src);
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip = fsd.getINodesInPath4Write(src);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index c64dfea..e9514ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -64,11 +64,9 @@ class FSDirRenameOp {
}
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
- byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
HdfsFileStatus resultingStat = null;
- src = fsd.resolvePath(pc, src, srcComponents);
- dst = fsd.resolvePath(pc, dst, dstComponents);
+ src = fsd.resolvePath(pc, src);
+ dst = fsd.resolvePath(pc, dst);
@SuppressWarnings("deprecation")
final boolean status = renameTo(fsd, pc, src, dst, logRetryCache);
if (status) {
@@ -239,11 +237,9 @@ class FSDirRenameOp {
}
final FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] srcComponents = FSDirectory.getPathComponentsForReservedPath(src);
- byte[][] dstComponents = FSDirectory.getPathComponentsForReservedPath(dst);
BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- src = fsd.resolvePath(pc, src, srcComponents);
- dst = fsd.resolvePath(pc, dst, dstComponents);
+ src = fsd.resolvePath(pc, src);
+ dst = fsd.resolvePath(pc, dst);
renameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options);
INodesInPath dstIIP = fsd.getINodesInPath(dst, false);
HdfsFileStatus resultingStat = fsd.getAuditFileInfo(dstIIP);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 9fbc9f4..40efc8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -52,24 +52,20 @@ import static org.apache.hadoop.util.Time.now;
class FSDirStatAndListingOp {
static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg,
byte[] startAfter, boolean needLocation) throws IOException {
- byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(srcArg);
final String startAfterString = DFSUtil.bytes2String(startAfter);
String src = null;
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, srcArg, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
} else {
- src = FSDirectory.resolvePath(srcArg, pathComponents, fsd);
+ src = FSDirectory.resolvePath(srcArg, fsd);
}
// Get file name when startAfter is an INodePath
if (FSDirectory.isReservedName(startAfterString)) {
- byte[][] startAfterComponents = FSDirectory
- .getPathComponentsForReservedPath(startAfterString);
try {
- String tmp = FSDirectory.resolvePath(src, startAfterComponents, fsd);
+ String tmp = FSDirectory.resolvePath(startAfterString, fsd);
byte[][] regularPath = INode.getPathComponents(tmp);
startAfter = regularPath[regularPath.length - 1];
} catch (IOException e) {
@@ -110,14 +106,13 @@ class FSDirStatAndListingOp {
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException("Invalid file name: " + src);
}
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
} else {
- src = FSDirectory.resolvePath(src, pathComponents, fsd);
+ src = FSDirectory.resolvePath(srcArg, fsd);
}
return getFileInfo(fsd, src, FSDirectory.isReservedRawName(srcArg),
resolveLink);
@@ -128,8 +123,7 @@ class FSDirStatAndListingOp {
*/
static boolean isFileClosed(FSDirectory fsd, String src) throws IOException {
FSPermissionChecker pc = fsd.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkTraverse(pc, iip);
@@ -139,9 +133,8 @@ class FSDirStatAndListingOp {
static ContentSummary getContentSummary(
FSDirectory fsd, String src) throws IOException {
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,
@@ -164,11 +157,10 @@ class FSDirStatAndListingOp {
"Negative length is not supported. File: " + src);
CacheManager cm = fsd.getFSNamesystem().getCacheManager();
BlockManager bm = fsd.getBlockManager();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean isReservedName = FSDirectory.isReservedRawName(src);
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (fsd.isPermissionEnabled()) {
@@ -628,12 +620,11 @@ class FSDirStatAndListingOp {
static QuotaUsage getQuotaUsage(
FSDirectory fsd, String src) throws IOException {
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = fsd.getPermissionChecker();
final INodesInPath iip;
fsd.readLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath(src, false);
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index 44a171a..e78c7b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -53,11 +53,10 @@ class FSDirSymlinkOp {
}
FSPermissionChecker pc = fsn.getPermissionChecker();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(link);
INodesInPath iip;
fsd.writeLock();
try {
- link = fsd.resolvePath(pc, link, pathComponents);
+ link = fsd.resolvePath(pc, link);
iip = fsd.getINodesInPath4Write(link, false);
if (!createParent) {
fsd.verifyParentDir(iip, link);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index 29c663a..8ac59f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -71,15 +71,13 @@ final class FSDirTruncateOp {
assert fsn.hasWriteLock();
FSDirectory fsd = fsn.getFSDirectory();
- byte[][] pathComponents = FSDirectory
- .getPathComponentsForReservedPath(srcArg);
final String src;
final INodesInPath iip;
final boolean onBlockBoundary;
Block truncateBlock = null;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, srcArg, pathComponents);
+ src = fsd.resolvePath(pc, srcArg);
iip = fsd.getINodesInPath4Write(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 2e5893f..0f55875 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -119,8 +119,7 @@ class FSDirWriteFileOp {
static void abandonBlock(
FSDirectory fsd, FSPermissionChecker pc, ExtendedBlock b, long fileId,
String src, String holder) throws IOException {
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INode inode;
final INodesInPath iip;
@@ -185,8 +184,7 @@ class FSDirWriteFileOp {
String clientMachine;
final boolean isStriped;
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsn.dir.resolvePath(pc, src, pathComponents);
+ src = fsn.dir.resolvePath(pc, src);
FileState fileState = analyzeFileState(fsn, src, fileId, clientName,
previous, onRetryBlock);
if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
@@ -359,8 +357,7 @@ class FSDirWriteFileOp {
boolean isRawPath = FSDirectory.isReservedRawName(src);
FSDirectory fsd = fsn.getFSDirectory();
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip = fsd.getINodesInPath4Write(src);
// Verify that the destination does not exist as a directory already.
@@ -469,9 +466,8 @@ class FSDirWriteFileOp {
FSPermissionChecker pc, String src,
CryptoProtocolVersion[] supportedVersions)
throws IOException {
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSDirectory fsd = fsn.getFSDirectory();
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
INodesInPath iip = fsd.getINodesInPath4Write(src);
// Nothing to do if the path is not within an EZ
final EncryptionZone zone = FSDirEncryptionZoneOp.getEZForPath(fsd, iip);
@@ -738,8 +734,7 @@ class FSDirWriteFileOp {
src + " for " + holder);
}
checkBlock(fsn, last);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsn.dir.resolvePath(pc, src, pathComponents);
+ src = fsn.dir.resolvePath(pc, src);
return completeFileInternal(fsn, src, holder,
ExtendedBlock.getLocalBlock(last), fileId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 92686c5..668e9e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -67,8 +67,7 @@ class FSDirXAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
INodesInPath iip;
@@ -95,8 +94,7 @@ class FSDirXAttrOp {
if (!getAll) {
XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
}
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
@@ -136,8 +134,7 @@ class FSDirXAttrOp {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
final FSPermissionChecker pc = fsd.getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
/* To access xattr names, you need EXECUTE in the owning directory. */
@@ -164,15 +161,13 @@ class FSDirXAttrOp {
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
- src);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
INodesInPath iip;
fsd.writeLock();
try {
- src = fsd.resolvePath(pc, src, pathComponents);
+ src = fsd.resolvePath(pc, src);
iip = fsd.getINodesInPath4Write(src);
checkXAttrChangeAccess(fsd, iip, xAttr, pc);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 11a7899..8321fda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -124,6 +124,8 @@ public class FSDirectory implements Closeable {
public final static String DOT_INODES_STRING = ".inodes";
public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING);
+ private final static byte[] DOT_DOT =
+ DFSUtil.string2Bytes("..");
public final static HdfsFileStatus DOT_RESERVED_STATUS =
new HdfsFileStatus(0, true, 0, 0, 0, 0, new FsPermission((short) 01770),
@@ -517,7 +519,6 @@ public class FSDirectory implements Closeable {
*
* @param pc The permission checker used when resolving path.
* @param path The path to resolve.
- * @param pathComponents path components corresponding to the path
* @return if the path indicates an inode, return path after replacing up to
* <inodeid> with the corresponding path of the inode, else the path
* in {@code src} as is. If the path refers to a path in the "raw"
@@ -525,12 +526,12 @@ public class FSDirectory implements Closeable {
* @throws FileNotFoundException
* @throws AccessControlException
*/
- String resolvePath(FSPermissionChecker pc, String path, byte[][] pathComponents)
+ String resolvePath(FSPermissionChecker pc, String path)
throws FileNotFoundException, AccessControlException {
if (isReservedRawName(path) && isPermissionEnabled) {
pc.checkSuperuserPrivilege();
}
- return resolvePath(path, pathComponents, this);
+ return resolvePath(path, this);
}
/**
@@ -1289,13 +1290,6 @@ public class FSDirectory implements Closeable {
return components.toArray(new byte[components.size()][]);
}
- /**
- * @return path components for reserved path, else null.
- */
- static byte[][] getPathComponentsForReservedPath(String src) {
- return !isReservedName(src) ? null : INode.getPathComponents(src);
- }
-
/** Check if a given inode name is reserved */
public static boolean isReservedName(INode inode) {
return CHECK_RESERVED_FILE_NAMES
@@ -1321,6 +1315,12 @@ public class FSDirectory implements Closeable {
Path.SEPARATOR + DOT_INODES_STRING);
}
+ static boolean isReservedName(byte[][] components) {
+ return (components.length > 2) &&
+ Arrays.equals(INodeDirectory.ROOT_NAME, components[0]) &&
+ Arrays.equals(DOT_RESERVED, components[1]);
+ }
+
/**
* Resolve a /.reserved/... path to a non-reserved path.
* <p/>
@@ -1340,7 +1340,6 @@ public class FSDirectory implements Closeable {
* unencrypted file).
*
* @param src path that is being processed
- * @param pathComponents path components corresponding to the path
* @param fsd FSDirectory
* @return if the path indicates an inode, return path after replacing up to
* <inodeid> with the corresponding path of the inode, else the path
@@ -1348,45 +1347,37 @@ public class FSDirectory implements Closeable {
* directory, return the non-raw pathname.
* @throws FileNotFoundException if inodeid is invalid
*/
- static String resolvePath(String src, byte[][] pathComponents,
+ static String resolvePath(String src,
FSDirectory fsd) throws FileNotFoundException {
- final int nComponents = (pathComponents == null) ?
- 0 : pathComponents.length;
- if (nComponents <= 2) {
- return src;
- }
- if (!Arrays.equals(DOT_RESERVED, pathComponents[1])) {
+ byte[][] pathComponents = INode.getPathComponents(src);
+ final int nComponents = pathComponents.length;
+ if (!isReservedName(pathComponents)) {
/* This is not a /.reserved/ path so do nothing. */
- return src;
- }
-
- if (Arrays.equals(DOT_INODES, pathComponents[2])) {
+ } else if (Arrays.equals(DOT_INODES, pathComponents[2])) {
/* It's a /.reserved/.inodes path. */
if (nComponents > 3) {
- return resolveDotInodesPath(src, pathComponents, fsd);
- } else {
- return src;
+ pathComponents = resolveDotInodesPath(pathComponents, fsd);
}
} else if (Arrays.equals(RAW, pathComponents[2])) {
/* It's /.reserved/raw so strip off the /.reserved/raw prefix. */
if (nComponents == 3) {
- return Path.SEPARATOR;
+ pathComponents = new byte[][]{INodeDirectory.ROOT_NAME};
} else {
if (nComponents == 4
&& Arrays.equals(DOT_RESERVED, pathComponents[3])) {
/* It's /.reserved/raw/.reserved so don't strip */
- return src;
} else {
- return constructRemainingPath("", pathComponents, 3);
+ pathComponents = constructRemainingPath(
+ new byte[][]{INodeDirectory.ROOT_NAME}, pathComponents, 3);
}
}
- } else {
- /* It's some sort of /.reserved/<unknown> path. Ignore it. */
- return src;
}
+ // this double conversion will be unnecessary when resolving returns
+ // INodesInPath (needs components byte[][])
+ return DFSUtil.byteArray2PathString(pathComponents);
}
- private static String resolveDotInodesPath(String src,
+ private static byte[][] resolveDotInodesPath(
byte[][] pathComponents, FSDirectory fsd)
throws FileNotFoundException {
final String inodeId = DFSUtil.bytes2String(pathComponents[3]);
@@ -1394,48 +1385,47 @@ public class FSDirectory implements Closeable {
try {
id = Long.parseLong(inodeId);
} catch (NumberFormatException e) {
- throw new FileNotFoundException("Invalid inode path: " + src);
+ throw new FileNotFoundException("Invalid inode path: " +
+ DFSUtil.byteArray2PathString(pathComponents));
}
if (id == INodeId.ROOT_INODE_ID && pathComponents.length == 4) {
- return Path.SEPARATOR;
+ return new byte[][]{INodeDirectory.ROOT_NAME};
}
INode inode = fsd.getInode(id);
if (inode == null) {
throw new FileNotFoundException(
- "File for given inode path does not exist: " + src);
+ "File for given inode path does not exist: " +
+ DFSUtil.byteArray2PathString(pathComponents));
}
-
+
// Handle single ".." for NFS lookup support.
if ((pathComponents.length > 4)
- && DFSUtil.bytes2String(pathComponents[4]).equals("..")) {
+ && Arrays.equals(pathComponents[4], DOT_DOT)) {
INode parent = inode.getParent();
if (parent == null || parent.getId() == INodeId.ROOT_INODE_ID) {
// inode is root, or its parent is root.
- return Path.SEPARATOR;
- } else {
- return parent.getFullPathName();
+ return new byte[][]{INodeDirectory.ROOT_NAME};
}
+ return parent.getPathComponents();
}
-
- String path = "";
- if (id != INodeId.ROOT_INODE_ID) {
- path = inode.getFullPathName();
- }
- return constructRemainingPath(path, pathComponents, 4);
+ return constructRemainingPath(
+ inode.getPathComponents(), pathComponents, 4);
}
- private static String constructRemainingPath(String pathPrefix,
- byte[][] pathComponents, int startAt) {
-
- StringBuilder path = new StringBuilder(pathPrefix);
- for (int i = startAt; i < pathComponents.length; i++) {
- path.append(Path.SEPARATOR).append(
- DFSUtil.bytes2String(pathComponents[i]));
+ private static byte[][] constructRemainingPath(byte[][] components,
+ byte[][] extraComponents, int startAt) {
+ int remainder = extraComponents.length - startAt;
+ if (remainder > 0) {
+ // grow the array and copy in the remaining components
+ int pos = components.length;
+ components = Arrays.copyOf(components, pos + remainder);
+ System.arraycopy(extraComponents, startAt, components, pos, remainder);
}
if (NameNode.LOG.isDebugEnabled()) {
- NameNode.LOG.debug("Resolved path is " + path);
+ NameNode.LOG.debug(
+ "Resolved path is " + DFSUtil.byteArray2PathString(components));
}
- return path.toString();
+ return components;
}
INode getINode4DotSnapshot(String src) throws UnresolvedLinkException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index fb87d4c..be084c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1790,8 +1790,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
logAuditEvent(true, "open", srcArg);
if (!isInSafeMode() && res.updateAccessTime()) {
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
- srcArg);
String src = srcArg;
writeLock();
final long now = now();
@@ -1815,7 +1813,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* HDFS-7463. A better fix is to change the edit log of SetTime to
* use inode id instead of a path.
*/
- src = dir.resolvePath(pc, srcArg, pathComponents);
+ src = dir.resolvePath(pc, srcArg);
final INodesInPath iip = dir.getINodesInPath(src, true);
INode inode = iip.getLastINode();
boolean updateAccessTime = inode != null &&
@@ -2309,12 +2307,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
boolean skipSync = false;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot recover the lease of " + src);
- src = dir.resolvePath(pc, src, pathComponents);
+ src = dir.resolvePath(pc, src);
final INodesInPath iip = dir.getINodesInPath4Write(src);
final INodeFile inode = INodeFile.valueOf(iip.getLastINode(), src);
if (!inode.isUnderConstruction()) {
@@ -2553,14 +2550,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final List<DatanodeStorageInfo> chosen;
final boolean isStriped;
checkOperation(OperationCategory.READ);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
readLock();
try {
checkOperation(OperationCategory.READ);
//check safe mode
checkNameNodeSafeMode("Cannot add datanode; src=" + src + ", blk=" + blk);
- src = dir.resolvePath(pc, src, pathComponents);
+ src = dir.resolvePath(pc, src);
//check lease
final INode inode;
@@ -3073,14 +3069,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot fsync file " + src);
- src = dir.resolvePath(pc, src, pathComponents);
+ src = dir.resolvePath(pc, src);
final INode inode;
if (fileId == HdfsConstants.GRANDFATHER_INODE_ID) {
// Older clients may not have given us an inode ID to work with.
@@ -6878,11 +6873,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void checkAccess(String src, FsAction mode) throws IOException {
checkOperation(OperationCategory.READ);
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
readLock();
try {
checkOperation(OperationCategory.READ);
- src = FSDirectory.resolvePath(src, pathComponents, dir);
+ src = FSDirectory.resolvePath(src, dir);
final INodesInPath iip = dir.getINodesInPath(src, true);
INode inode = iip.getLastINode();
if (inode == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03dea65e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index 731ff95..3492c2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -857,9 +857,9 @@ public class TestINodeFile {
byte[][] actual = FSDirectory.getPathComponents(inode);
DFSTestUtil.checkComponentsEquals(expected, actual);
}
-
+
/**
- * Tests for {@link FSDirectory#resolvePath(String, byte[][], FSDirectory)}
+ * Tests for {@link FSDirectory#resolvePath(String, FSDirectory)}
*/
@Test
public void testInodePath() throws IOException {
@@ -869,60 +869,53 @@ public class TestINodeFile {
// For an any inode look up return inode corresponding to "c" from /a/b/c
FSDirectory fsd = Mockito.mock(FSDirectory.class);
Mockito.doReturn(inode).when(fsd).getInode(Mockito.anyLong());
-
- // Null components
- assertEquals("/test", FSDirectory.resolvePath("/test", null, fsd));
-
+
// Tests for FSDirectory#resolvePath()
// Non inode regular path
- byte[][] components = INode.getPathComponents(path);
- String resolvedPath = FSDirectory.resolvePath(path, components, fsd);
+ String resolvedPath = FSDirectory.resolvePath(path, fsd);
assertEquals(path, resolvedPath);
-
+
// Inode path with no trailing separator
- components = INode.getPathComponents("/.reserved/.inodes/1");
- resolvedPath = FSDirectory.resolvePath(path, components, fsd);
+ String testPath = "/.reserved/.inodes/1";
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(path, resolvedPath);
-
+
// Inode path with trailing separator
- components = INode.getPathComponents("/.reserved/.inodes/1/");
+ testPath = "/.reserved/.inodes/1/";
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(path, resolvedPath);
-
+
// Inode relative path
- components = INode.getPathComponents("/.reserved/.inodes/1/d/e/f");
- resolvedPath = FSDirectory.resolvePath(path, components, fsd);
+ testPath = "/.reserved/.inodes/1/d/e/f";
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals("/a/b/c/d/e/f", resolvedPath);
-
+
// A path with just .inodes returns the path as is
- String testPath = "/.reserved/.inodes";
- components = INode.getPathComponents(testPath);
- resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
+ testPath = "/.reserved/.inodes";
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(testPath, resolvedPath);
-
+
// Root inode path
testPath = "/.reserved/.inodes/" + INodeId.ROOT_INODE_ID;
- components = INode.getPathComponents(testPath);
- resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals("/", resolvedPath);
-
+
// An invalid inode path should remain unresolved
testPath = "/.invalid/.inodes/1";
- components = INode.getPathComponents(testPath);
- resolvedPath = FSDirectory.resolvePath(testPath, components, fsd);
+ resolvedPath = FSDirectory.resolvePath(testPath, fsd);
assertEquals(testPath, resolvedPath);
-
+
// Test path with nonexistent(deleted or wrong id) inode
Mockito.doReturn(null).when(fsd).getInode(Mockito.anyLong());
testPath = "/.reserved/.inodes/1234";
- components = INode.getPathComponents(testPath);
try {
- String realPath = FSDirectory.resolvePath(testPath, components, fsd);
+ String realPath = FSDirectory.resolvePath(testPath, fsd);
fail("Path should not be resolved:" + realPath);
} catch (IOException e) {
assertTrue(e instanceof FileNotFoundException);
}
}
-
+
private static INodeDirectory getDir(final FSDirectory fsdir, final Path dir)
throws IOException {
final String dirStr = dir.toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-5476. Non existent
application reported as ACCEPTED by YarnClientImpl (Junping Du via Varun
Saxena)
Posted by cn...@apache.org.
YARN-5476. Non existent application reported as ACCEPTED by YarnClientImpl (Junping Du via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23c6e3c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23c6e3c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23c6e3c4
Branch: refs/heads/HADOOP-13345
Commit: 23c6e3c4e41fecc61d062542cb61e68898235006
Parents: 9019606
Author: Varun Saxena <va...@apache.org>
Authored: Fri Aug 12 20:37:58 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Aug 12 20:37:58 2016 +0530
----------------------------------------------------------------------
.../server/resourcemanager/rmapp/RMAppImpl.java | 16 +++++++--------
.../rmapp/TestRMAppTransitions.java | 21 +++++++++++++-------
2 files changed, 22 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c6e3c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 82e669a..e5bde32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -902,8 +902,9 @@ public class RMAppImpl implements RMApp, Recoverable {
//TODO recover collector address.
//this.collectorAddr = appState.getCollectorAddr();
- // send the ATS create Event
- sendATSCreateEvent(this, this.startTime);
+ // send the ATS create Event during RM recovery.
+ // NOTE: it could be duplicated with events sent before RM get restarted.
+ sendATSCreateEvent();
RMAppAttemptImpl preAttempt = null;
for (ApplicationAttemptId attemptId :
new TreeSet<>(appState.attempts.keySet())) {
@@ -1134,6 +1135,8 @@ public class RMAppImpl implements RMApp, Recoverable {
public void transition(RMAppImpl app, RMAppEvent event) {
app.handler.handle(new AppAddedSchedulerEvent(app.user,
app.submissionContext, false));
+ // send the ATS create Event
+ app.sendATSCreateEvent();
}
}
@@ -1212,9 +1215,6 @@ public class RMAppImpl implements RMApp, Recoverable {
// communication
LOG.info("Storing application with id " + app.applicationId);
app.rmContext.getStateStore().storeNewApplication(app);
-
- // send the ATS create Event
- app.sendATSCreateEvent(app, app.startTime);
}
}
@@ -1912,9 +1912,9 @@ public class RMAppImpl implements RMApp, Recoverable {
return callerContext;
}
- private void sendATSCreateEvent(RMApp app, long startTime) {
- rmContext.getRMApplicationHistoryWriter().applicationStarted(app);
- rmContext.getSystemMetricsPublisher().appCreated(app, startTime);
+ private void sendATSCreateEvent() {
+ rmContext.getRMApplicationHistoryWriter().applicationStarted(this);
+ rmContext.getSystemMetricsPublisher().appCreated(this, this.startTime);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c6e3c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 9d2f89d..6bf5ecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -373,25 +374,32 @@ public class TestRMAppTransitions {
protected RMApp testCreateAppNewSaving(
ApplicationSubmissionContext submissionContext) throws IOException {
- RMApp application = createNewTestApp(submissionContext);
+ RMApp application = createNewTestApp(submissionContext);
// NEW => NEW_SAVING event RMAppEventType.START
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.START);
application.handle(event);
assertStartTimeSet(application);
assertAppState(RMAppState.NEW_SAVING, application);
+ // verify sendATSCreateEvent() is not get called during
+ // RMAppNewlySavingTransition.
+ verify(publisher, times(0)).appCreated(eq(application), anyLong());
return application;
}
protected RMApp testCreateAppSubmittedNoRecovery(
ApplicationSubmissionContext submissionContext) throws IOException {
- RMApp application = testCreateAppNewSaving(submissionContext);
- // NEW_SAVING => SUBMITTED event RMAppEventType.APP_SAVED
+ RMApp application = testCreateAppNewSaving(submissionContext);
+ // NEW_SAVING => SUBMITTED event RMAppEventType.APP_NEW_SAVED
RMAppEvent event =
- new RMAppEvent(application.getApplicationId(), RMAppEventType.APP_NEW_SAVED);
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.APP_NEW_SAVED);
application.handle(event);
assertStartTimeSet(application);
assertAppState(RMAppState.SUBMITTED, application);
+ // verify sendATSCreateEvent() is get called during
+ // AddApplicationToSchedulerTransition.
+ verify(publisher).appCreated(eq(application), anyLong());
return application;
}
@@ -406,7 +414,6 @@ public class TestRMAppTransitions {
RMAppEvent event =
new RMAppRecoverEvent(application.getApplicationId(), state);
-
application.handle(event);
assertStartTimeSet(application);
assertAppState(RMAppState.SUBMITTED, application);
@@ -416,7 +423,7 @@ public class TestRMAppTransitions {
protected RMApp testCreateAppAccepted(
ApplicationSubmissionContext submissionContext) throws IOException {
RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
- // SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
+ // SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
RMAppEvent event =
new RMAppEvent(application.getApplicationId(),
RMAppEventType.APP_ACCEPTED);
@@ -428,7 +435,7 @@ public class TestRMAppTransitions {
protected RMApp testCreateAppRunning(
ApplicationSubmissionContext submissionContext) throws IOException {
- RMApp application = testCreateAppAccepted(submissionContext);
+ RMApp application = testCreateAppAccepted(submissionContext);
// ACCEPTED => RUNNING event RMAppEventType.ATTEMPT_REGISTERED
RMAppEvent event =
new RMAppEvent(application.getApplicationId(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-4833. For Queue
AccessControlException client retries multiple times on both RM. Contributed
by Bibin A Chundatt
Posted by cn...@apache.org.
YARN-4833. For Queue AccessControlException client retries multiple times on both RM. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/874577a6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/874577a6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/874577a6
Branch: refs/heads/HADOOP-13345
Commit: 874577a67df8a49243586909d866034df4e3b276
Parents: db719ef
Author: Naganarasimha <na...@apache.org>
Authored: Fri Aug 12 01:08:24 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Aug 12 01:09:41 2016 +0530
----------------------------------------------------------------------
.../server/resourcemanager/RMAppManager.java | 9 ++--
.../server/resourcemanager/TestAppManager.java | 43 ++++++++++++++++++++
2 files changed, 47 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874577a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 49daedb..136dee0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -285,7 +285,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
@SuppressWarnings("unchecked")
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user) throws YarnException, AccessControlException {
+ String user) throws YarnException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Passing start time as -1. It will be eventually set in RMAppImpl
@@ -336,8 +336,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
- String user, boolean isRecovery, long startTime)
- throws YarnException, AccessControlException {
+ String user, boolean isRecovery, long startTime) throws YarnException {
// Do queue mapping
if (!isRecovery) {
if (rmContext.getQueuePlacementManager() != null) {
@@ -380,9 +379,9 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
SchedulerUtils.toAccessType(QueueACL.ADMINISTER_QUEUE),
applicationId.toString(), appName, Server.getRemoteAddress(),
null))) {
- throw new AccessControlException(
+ throw RPCUtil.getRemoteException(new AccessControlException(
"User " + user + " does not have permission to submit "
- + applicationId + " to queue " + submissionContext.getQueue());
+ + applicationId + " to queue " + submissionContext.getQueue()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/874577a6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index c3d04de..68f1e22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -39,14 +39,18 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -69,9 +73,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessM
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@@ -281,6 +287,43 @@ public class TestAppManager{
}
@Test
+ public void testQueueSubmitWithNoPermission() throws IOException {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getCanonicalName());
+ conf.set("yarn.scheduler.capacity.root.acl_submit_applications", " ");
+ conf.set("yarn.scheduler.capacity.root.acl_administer_queue", " ");
+
+ conf.set("yarn.scheduler.capacity.root.default.acl_submit_applications",
+ " ");
+ conf.set("yarn.scheduler.capacity.root.default.acl_administer_queue", " ");
+ conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
+ MockRM mockRM = new MockRM(conf);
+ ClientRMService rmService = mockRM.getClientRMService();
+ SubmitApplicationRequest req =
+ Records.newRecord(SubmitApplicationRequest.class);
+ ApplicationSubmissionContext sub =
+ Records.newRecord(ApplicationSubmissionContext.class);
+ sub.setApplicationId(appId);
+ ResourceRequest resReg =
+ ResourceRequest.newInstance(Priority.newInstance(0),
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
+ sub.setAMContainerResourceRequest(resReg);
+ req.setApplicationSubmissionContext(sub);
+ try {
+ rmService.submitApplication(req);
+ } catch (Exception e) {
+ if (e instanceof YarnException) {
+ Assert.assertTrue(e.getCause() instanceof AccessControlException);
+ } else {
+ Assert.fail("Yarn exception is expected");
+ }
+ } finally {
+ mockRM.close();
+ }
+ }
+
+ @Test
public void testRMAppRetireSome() throws Exception {
long now = System.currentTimeMillis();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-8668. Erasure Coding: revisit
buffer used for encoding and decoding. Contributed by Sammi Chen
Posted by cn...@apache.org.
HDFS-8668. Erasure Coding: revisit buffer used for encoding and decoding. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5af9be7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5af9be7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5af9be7
Branch: refs/heads/HADOOP-13345
Commit: b5af9be72c72734d668f817c99d889031922a951
Parents: 4d3ea92
Author: Kai Zheng <ka...@intel.com>
Authored: Sat Aug 13 13:52:37 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Sat Aug 13 13:52:37 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/io/ElasticByteBufferPool.java | 1 +
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 52 +++++++++++++++-----
.../hadoop/hdfs/DFSStripedInputStream.java | 18 ++++---
.../hadoop/hdfs/DFSStripedOutputStream.java | 29 +++++++++--
.../StripedBlockChecksumReconstructor.java | 3 +-
.../erasurecode/StripedBlockReader.java | 4 ++
.../erasurecode/StripedBlockReconstructor.java | 4 +-
.../erasurecode/StripedBlockWriter.java | 22 +++++++--
.../datanode/erasurecode/StripedReader.java | 11 ++++-
.../erasurecode/StripedReconstructor.java | 13 ++++-
.../datanode/erasurecode/StripedWriter.java | 8 +++
.../hadoop/hdfs/TestDFSStripedInputStream.java | 8 +++
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 8 +++
.../TestDFSStripedOutputStreamWithFailure.java | 11 ++++-
.../hadoop/hdfs/TestReconstructStripedFile.java | 11 ++++-
15 files changed, 169 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
index 694fcbe..c35d608 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
@@ -101,6 +101,7 @@ public final class ElasticByteBufferPool implements ByteBufferPool {
@Override
public synchronized void putBuffer(ByteBuffer buffer) {
+ buffer.clear();
TreeMap<Key, ByteBuffer> tree = getBufferTree(buffer.isDirect());
while (true) {
Key key = new Key(buffer.capacity(), System.nanoTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index cc919da..93aee0e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.util.EnumSet;
import java.util.concurrent.atomic.AtomicReference;
@@ -393,11 +394,47 @@ public class DFSOutputStream extends FSOutputSummer
@Override
protected synchronized void writeChunk(byte[] b, int offset, int len,
byte[] checksum, int ckoff, int cklen) throws IOException {
+ writeChunkPrepare(len, ckoff, cklen);
+
+ currentPacket.writeChecksum(checksum, ckoff, cklen);
+ currentPacket.writeData(b, offset, len);
+ currentPacket.incNumChunks();
+ getStreamer().incBytesCurBlock(len);
+
+ // If packet is full, enqueue it for transmission
+ if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() ||
+ getStreamer().getBytesCurBlock() == blockSize) {
+ enqueueCurrentPacketFull();
+ }
+ }
+
+ /* write the data chunk in <code>buffer</code> staring at
+ * <code>buffer.position</code> with
+ * a length of <code>len > 0</code>, and its checksum
+ */
+ protected synchronized void writeChunk(ByteBuffer buffer, int len,
+ byte[] checksum, int ckoff, int cklen) throws IOException {
+ writeChunkPrepare(len, ckoff, cklen);
+
+ currentPacket.writeChecksum(checksum, ckoff, cklen);
+ currentPacket.writeData(buffer, len);
+ currentPacket.incNumChunks();
+ getStreamer().incBytesCurBlock(len);
+
+ // If packet is full, enqueue it for transmission
+ if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() ||
+ getStreamer().getBytesCurBlock() == blockSize) {
+ enqueueCurrentPacketFull();
+ }
+ }
+
+ private synchronized void writeChunkPrepare(int buflen,
+ int ckoff, int cklen) throws IOException {
dfsClient.checkOpen();
checkClosed();
- if (len > bytesPerChecksum) {
- throw new IOException("writeChunk() buffer size is " + len +
+ if (buflen > bytesPerChecksum) {
+ throw new IOException("writeChunk() buffer size is " + buflen +
" is larger than supported bytesPerChecksum " +
bytesPerChecksum);
}
@@ -414,17 +451,6 @@ public class DFSOutputStream extends FSOutputSummer
currentPacket.getSeqno(), src, packetSize, chunksPerPacket,
getStreamer().getBytesCurBlock() + ", " + this);
}
-
- currentPacket.writeChecksum(checksum, ckoff, cklen);
- currentPacket.writeData(b, offset, len);
- currentPacket.incNumChunks();
- getStreamer().incBytesCurBlock(len);
-
- // If packet is full, enqueue it for transmission
- if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() ||
- getStreamer().getBytesCurBlock() == blockSize) {
- enqueueCurrentPacketFull();
- }
}
void enqueueCurrentPacket() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 1bdbc32..d93863c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -35,12 +35,12 @@ import static org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult;
+import org.apache.hadoop.io.ElasticByteBufferPool;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
-import org.apache.hadoop.util.DirectBufferPool;
import java.io.EOFException;
import java.io.IOException;
@@ -139,7 +139,7 @@ public class DFSStripedInputStream extends DFSInputStream {
}
}
- private static final DirectBufferPool bufferPool = new DirectBufferPool();
+ private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
private final BlockReaderInfo[] blockReaders;
private final int cellSize;
@@ -194,9 +194,14 @@ public class DFSStripedInputStream extends DFSInputStream {
}
}
+ private boolean useDirectBuffer() {
+ return decoder.preferDirectBuffer();
+ }
+
private void resetCurStripeBuffer() {
if (curStripeBuf == null) {
- curStripeBuf = bufferPool.getBuffer(cellSize * dataBlkNum);
+ curStripeBuf = BUFFER_POOL.getBuffer(useDirectBuffer(),
+ cellSize * dataBlkNum);
}
curStripeBuf.clear();
curStripeRange = new StripeRange(0, 0);
@@ -204,7 +209,8 @@ public class DFSStripedInputStream extends DFSInputStream {
private ByteBuffer getParityBuffer() {
if (parityBuf == null) {
- parityBuf = bufferPool.getBuffer(cellSize * parityBlkNum);
+ parityBuf = BUFFER_POOL.getBuffer(useDirectBuffer(),
+ cellSize * parityBlkNum);
}
parityBuf.clear();
return parityBuf;
@@ -235,11 +241,11 @@ public class DFSStripedInputStream extends DFSInputStream {
public synchronized void close() throws IOException {
super.close();
if (curStripeBuf != null) {
- bufferPool.returnBuffer(curStripeBuf);
+ BUFFER_POOL.putBuffer(curStripeBuf);
curStripeBuf = null;
}
if (parityBuf != null) {
- bufferPool.returnBuffer(parityBuf);
+ BUFFER_POOL.putBuffer(parityBuf);
parityBuf = null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 85dc749..502e0a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.ByteBufferPool;
+import org.apache.hadoop.io.ElasticByteBufferPool;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -75,6 +77,9 @@ import org.apache.htrace.core.TraceScope;
*/
@InterfaceAudience.Private
public class DFSStripedOutputStream extends DFSOutputStream {
+
+ private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
+
static class MultipleBlockingQueue<T> {
private final List<BlockingQueue<T>> queues;
@@ -208,7 +213,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
buffers = new ByteBuffer[numAllBlocks];
for (int i = 0; i < buffers.length; i++) {
- buffers[i] = ByteBuffer.wrap(byteArrayManager.newByteArray(cellSize));
+ buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize);
}
}
@@ -236,7 +241,10 @@ public class DFSStripedOutputStream extends DFSOutputStream {
private void release() {
for (int i = 0; i < numAllBlocks; i++) {
- byteArrayManager.release(buffers[i].array());
+ if (buffers[i] != null) {
+ BUFFER_POOL.putBuffer(buffers[i]);
+ buffers[i] = null;
+ }
}
}
@@ -311,6 +319,10 @@ public class DFSStripedOutputStream extends DFSOutputStream {
setCurrentStreamer(0);
}
+ private boolean useDirectBuffer() {
+ return encoder.preferDirectBuffer();
+ }
+
StripedDataStreamer getStripedDataStreamer(int i) {
return streamers.get(i);
}
@@ -907,11 +919,20 @@ public class DFSStripedOutputStream extends DFSOutputStream {
if (current.isHealthy()) {
try {
DataChecksum sum = getDataChecksum();
- sum.calculateChunkedSums(buffer.array(), 0, len, checksumBuf, 0);
+ if (buffer.isDirect()) {
+ ByteBuffer directCheckSumBuf =
+ BUFFER_POOL.getBuffer(true, checksumBuf.length);
+ sum.calculateChunkedSums(buffer, directCheckSumBuf);
+ directCheckSumBuf.get(checksumBuf);
+ BUFFER_POOL.putBuffer(directCheckSumBuf);
+ } else {
+ sum.calculateChunkedSums(buffer.array(), 0, len, checksumBuf, 0);
+ }
+
for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
- super.writeChunk(buffer.array(), i, chunkLen, checksumBuf, ckOffset,
+ super.writeChunk(buffer, chunkLen, checksumBuf, ckOffset,
getChecksumSize());
}
} catch(Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
index c7294c7..944ed9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java
@@ -57,6 +57,7 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
}
private void init() throws IOException {
+ initDecoderIfNecessary();
getStripedReader().init();
// allocate buffer to keep the reconstructed block data
targetBuffer = allocateBuffer(getBufferSize());
@@ -150,8 +151,6 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor {
}
private void reconstructTargets(int toReconstructLen) {
- initDecoderIfNecessary();
-
ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
ByteBuffer[] outputs = new ByteBuffer[1];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index 140c658..8f976c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -90,6 +90,10 @@ class StripedBlockReader {
return buffer;
}
+ void freeReadBuffer() {
+ buffer = null;
+ }
+
void resetBlockReader(long offsetInBlock) {
this.blockReader = createBlockReader(offsetInBlock);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index b800bef..9f9f15d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -49,6 +49,8 @@ class StripedBlockReconstructor extends StripedReconstructor
public void run() {
getDatanode().incrementXmitsInProgress();
try {
+ initDecoderIfNecessary();
+
getStripedReader().init();
stripedWriter.init();
@@ -96,8 +98,6 @@ class StripedBlockReconstructor extends StripedReconstructor
}
private void reconstructTargets(int toReconstructLen) {
- initDecoderIfNecessary();
-
ByteBuffer[] inputs = getStripedReader().getInputBuffers(toReconstructLen);
int[] erasedIndices = stripedWriter.getRealTargetIndices();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 32e8843..11551e7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.io.ByteBufferPool;
+import org.apache.hadoop.io.ElasticByteBufferPool;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.token.Token;
@@ -65,6 +67,7 @@ class StripedBlockWriter {
private ByteBuffer targetBuffer;
private long blockOffset4Target = 0;
private long seqNo4Target = 0;
+ private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
StripedBlockWriter(StripedWriter stripedWriter, DataNode datanode,
Configuration conf, ExtendedBlock block,
@@ -87,6 +90,10 @@ class StripedBlockWriter {
return targetBuffer;
}
+ void freeTargetBuffer() {
+ targetBuffer = null;
+ }
+
/**
* Initialize output/input streams for transferring data to target
* and send create block request.
@@ -154,9 +161,18 @@ class StripedBlockWriter {
return;
}
- stripedWriter.getChecksum().calculateChunkedSums(
- targetBuffer.array(), 0, targetBuffer.remaining(),
- stripedWriter.getChecksumBuf(), 0);
+ if (targetBuffer.isDirect()) {
+ ByteBuffer directCheckSumBuf =
+ BUFFER_POOL.getBuffer(true, stripedWriter.getChecksumBuf().length);
+ stripedWriter.getChecksum().calculateChunkedSums(
+ targetBuffer, directCheckSumBuf);
+ directCheckSumBuf.get(stripedWriter.getChecksumBuf());
+ BUFFER_POOL.putBuffer(directCheckSumBuf);
+ } else {
+ stripedWriter.getChecksum().calculateChunkedSums(
+ targetBuffer.array(), 0, targetBuffer.remaining(),
+ stripedWriter.getChecksumBuf(), 0);
+ }
int ckOff = 0;
while (targetBuffer.remaining() > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
index e6d4ceb..238c628 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReader.java
@@ -180,7 +180,7 @@ class StripedReader {
}
protected ByteBuffer allocateReadBuffer() {
- return ByteBuffer.allocate(getBufferSize());
+ return reconstructor.allocateBuffer(getBufferSize());
}
private void initZeroStrip() {
@@ -421,7 +421,16 @@ class StripedReader {
}
void close() {
+ if (zeroStripeBuffers != null) {
+ for (ByteBuffer zeroStripeBuffer : zeroStripeBuffers) {
+ reconstructor.freeBuffer(zeroStripeBuffer);
+ }
+ }
+ zeroStripeBuffers = null;
+
for (StripedBlockReader reader : readers) {
+ reconstructor.freeBuffer(reader.getReadBuffer());
+ reader.freeReadBuffer();
reader.closeBlockReader();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index 782d091..5641c35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.ByteBufferPool;
+import org.apache.hadoop.io.ElasticByteBufferPool;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
@@ -102,6 +104,7 @@ abstract class StripedReconstructor {
private final ErasureCodingPolicy ecPolicy;
private RawErasureDecoder decoder;
private final ExtendedBlock blockGroup;
+ private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
// position in striped internal block
private long positionInBlock;
@@ -139,8 +142,16 @@ abstract class StripedReconstructor {
*/
abstract void reconstruct() throws IOException;
+ boolean useDirectBuffer() {
+ return decoder.preferDirectBuffer();
+ }
+
ByteBuffer allocateBuffer(int length) {
- return ByteBuffer.allocate(length);
+ return BUFFER_POOL.getBuffer(useDirectBuffer(), length);
+ }
+
+ void freeBuffer(ByteBuffer buffer) {
+ BUFFER_POOL.putBuffer(buffer);
}
ExtendedBlock getBlock(int i) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
index ca7a3a8..c099bc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedWriter.java
@@ -297,6 +297,14 @@ class StripedWriter {
}
void close() {
+ for (StripedBlockWriter writer : writers) {
+ ByteBuffer targetBuffer = writer.getTargetBuffer();
+ if (targetBuffer != null) {
+ reconstructor.freeBuffer(targetBuffer);
+ writer.freeTargetBuffer();
+ }
+ }
+
for (int i = 0; i < targets.length; i++) {
writers[i].close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index a02a8d6..18c2de9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -32,7 +33,9 @@ import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.junit.After;
import org.junit.Assert;
@@ -77,6 +80,11 @@ public class TestDFSStripedInputStream {
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, INTERNAL_BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ if (ErasureCodeNative.isNativeCodeLoaded()) {
+ conf.set(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ }
SimulatedFSDataset.setFactory(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
DATA_BLK_NUM + PARITY_BLK_NUM).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index a12a8ce..8d54f08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -25,8 +25,11 @@ import java.util.Collections;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.After;
@@ -65,6 +68,11 @@ public class TestDFSStripedOutputStream {
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ if (ErasureCodeNative.isNativeCodeLoaded()) {
+ conf.set(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ }
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
fs = cluster.getFileSystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 83b6c58..11036a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -22,6 +22,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
@@ -183,6 +186,11 @@ public class TestDFSStripedOutputStreamWithFailure {
private void setup(Configuration conf) throws IOException {
final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ if (ErasureCodeNative.isNativeCodeLoaded()) {
+ conf.set(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ }
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.waitActive();
dfs = cluster.getFileSystem();
@@ -229,7 +237,8 @@ public class TestDFSStripedOutputStreamWithFailure {
final HdfsConfiguration conf = newHdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
- conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ conf.setInt(
+ CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
// Set short retry timeouts so this test runs faster
conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
for (int dn = 0; dn < NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; dn += 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5af9be7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 36d2dbd..59e9f87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -33,6 +33,7 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -47,6 +48,8 @@ import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCodeNative;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.After;
@@ -86,11 +89,17 @@ public class TestReconstructStripedFile {
public void setup() throws IOException {
final Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setInt(DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
+ conf.setInt(
+ DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY,
cellSize - 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
false);
+ if (ErasureCodeNative.isNativeCodeLoaded()) {
+ conf.set(
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
+ NativeRSRawErasureCoderFactory.class.getCanonicalName());
+ }
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
cluster.waitActive();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-5523. Yarn running container
log fetching causes OutOfMemoryError (Xuan Gong via Varun Saxena)
Posted by cn...@apache.org.
YARN-5523. Yarn running container log fetching causes OutOfMemoryError (Xuan Gong via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3037c56
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3037c56
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3037c56
Branch: refs/heads/HADOOP-13345
Commit: e3037c564117fe53742c130665b047dd17eff6d0
Parents: f80a729
Author: Varun Saxena <va...@apache.org>
Authored: Thu Aug 18 01:45:33 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Thu Aug 18 01:45:33 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 74 +++++++++++----
.../hadoop/yarn/client/cli/TestLogsCLI.java | 95 ++++++++++++++++++++
2 files changed, 150 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3037c56/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 908d379..25e3a46 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.client.cli;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
@@ -41,6 +42,7 @@ import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -462,15 +464,7 @@ public class LogsCLI extends Configured implements Tool {
PrintStream out = logCliHelper.createPrintStream(localDir, nodeId,
containerIdStr);
try {
- // fetch all the log files for the container
- // filter the log files based on the given -log_files pattern
- List<PerLogFileInfo> allLogFileInfos=
- getContainerLogFiles(getConf(), containerIdStr, nodeHttpAddress);
- List<String> fileNames = new ArrayList<String>();
- for (PerLogFileInfo fileInfo : allLogFileInfos) {
- fileNames.add(fileInfo.getFileName());
- }
- Set<String> matchedFiles = getMatchedLogFiles(request, fileNames,
+ Set<String> matchedFiles = getMatchedContainerLogFiles(request,
useRegex);
if (matchedFiles.isEmpty()) {
System.err.println("Can not find any log file matching the pattern: "
@@ -487,22 +481,33 @@ public class LogsCLI extends Configured implements Tool {
out.println(containerString);
out.println(StringUtils.repeat("=", containerString.length()));
boolean foundAnyLogs = false;
+ byte[] buffer = new byte[65536];
for (String logFile : newOptions.getLogTypes()) {
out.println("LogType:" + logFile);
out.println("Log Upload Time:"
+ Times.format(System.currentTimeMillis()));
out.println("Log Contents:");
+ InputStream is = null;
try {
- WebResource webResource =
- webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
- + nodeHttpAddress);
- ClientResponse response =
- webResource.path("ws").path("v1").path("node")
- .path("containers").path(containerIdStr).path("logs")
- .path(logFile)
- .queryParam("size", Long.toString(request.getBytes()))
- .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
- out.println(response.getEntity(String.class));
+ ClientResponse response = getResponeFromNMWebService(conf,
+ webServiceClient, request, logFile);
+ if (response != null && response.getStatusInfo().getStatusCode() ==
+ ClientResponse.Status.OK.getStatusCode()) {
+ is = response.getEntityInputStream();
+ int len = 0;
+ while((len = is.read(buffer)) != -1) {
+ out.write(buffer, 0, len);
+ }
+ out.println();
+ } else {
+ out.println("Can not get any logs for the log file: " + logFile);
+ String msg = "Response from the NodeManager:" + nodeId +
+ " WebService is " + ((response == null) ? "null":
+ "not successful," + " HTTP error code: " +
+ response.getStatus() + ", Server response:\n" +
+ response.getEntity(String.class));
+ out.println(msg);
+ }
StringBuilder sb = new StringBuilder();
sb.append("End of LogType:" + logFile + ".");
if (request.getContainerState() == ContainerState.RUNNING) {
@@ -517,6 +522,8 @@ public class LogsCLI extends Configured implements Tool {
System.err.println("Can not find the log file:" + logFile
+ " for the container:" + containerIdStr + " in NodeManager:"
+ nodeId);
+ } finally {
+ IOUtils.closeQuietly(is);
}
}
// for the case, we have already uploaded partial logs in HDFS
@@ -1189,4 +1196,33 @@ public class LogsCLI extends Configured implements Tool {
this.fileLength = fileLength;
}
}
+
+ @VisibleForTesting
+ public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
+ boolean useRegex) throws IOException {
+ // fetch all the log files for the container
+ // filter the log files based on the given -log_files pattern
+ List<PerLogFileInfo> allLogFileInfos=
+ getContainerLogFiles(getConf(), request.getContainerId(),
+ request.getNodeHttpAddress());
+ List<String> fileNames = new ArrayList<String>();
+ for (PerLogFileInfo fileInfo : allLogFileInfos) {
+ fileNames.add(fileInfo.getFileName());
+ }
+ return getMatchedLogFiles(request, fileNames,
+ useRegex);
+ }
+
+ @VisibleForTesting
+ public ClientResponse getResponeFromNMWebService(Configuration conf,
+ Client webServiceClient, ContainerLogsRequest request, String logFile) {
+ WebResource webResource =
+ webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
+ + request.getNodeHttpAddress());
+ return webResource.path("ws").path("v1").path("node")
+ .path("containers").path(request.getContainerId()).path("logs")
+ .path(logFile)
+ .queryParam("size", Long.toString(request.getBytes()))
+ .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3037c56/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 19ddb2f..a564f6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -30,10 +31,14 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -63,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
@@ -582,6 +588,95 @@ public class TestLogsCLI {
}
@Test (timeout = 5000)
+ public void testGetRunningContainerLogs() throws Exception {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId
+ .newInstance(appId, 1);
+
+ // Create a mock ApplicationAttempt Report
+ ApplicationAttemptReport mockAttemptReport = mock(
+ ApplicationAttemptReport.class);
+ doReturn(appAttemptId).when(mockAttemptReport).getApplicationAttemptId();
+ List<ApplicationAttemptReport> attemptReports = Arrays.asList(
+ mockAttemptReport);
+
+ // Create one mock containerReport
+ ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
+ ContainerReport mockContainerReport1 = mock(ContainerReport.class);
+ doReturn(containerId1).when(mockContainerReport1).getContainerId();
+ doReturn(nodeId).when(mockContainerReport1).getAssignedNode();
+ doReturn("http://localhost:2345").when(mockContainerReport1)
+ .getNodeHttpAddress();
+ doReturn(ContainerState.RUNNING).when(mockContainerReport1)
+ .getContainerState();
+ List<ContainerReport> containerReports = Arrays.asList(
+ mockContainerReport1);
+
+ // Mock the YarnClient, and it would report the previous created
+ // mockAttemptReport and previous two created mockContainerReports
+ YarnClient mockYarnClient = createMockYarnClient(
+ YarnApplicationState.RUNNING, ugi.getShortUserName(), true,
+ attemptReports, containerReports);
+ doReturn(mockContainerReport1).when(mockYarnClient).getContainerReport(
+ any(ContainerId.class));
+
+ // create local logs
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.get(configuration);
+ String rootLogDir = "target/LocalLogs";
+ Path rootLogDirPath = new Path(rootLogDir);
+ if (fs.exists(rootLogDirPath)) {
+ fs.delete(rootLogDirPath, true);
+ }
+ assertTrue(fs.mkdirs(rootLogDirPath));
+
+ Path appLogsDir = new Path(rootLogDirPath, appId.toString());
+ if (fs.exists(appLogsDir)) {
+ fs.delete(appLogsDir, true);
+ }
+ assertTrue(fs.mkdirs(appLogsDir));
+
+ String fileName = "syslog";
+ List<String> logTypes = new ArrayList<String>();
+ logTypes.add(fileName);
+ // create container logs in localLogDir
+ createContainerLogInLocalDir(appLogsDir, containerId1, fs, logTypes);
+
+ Path containerDirPath = new Path(appLogsDir, containerId1.toString());
+ Path logPath = new Path(containerDirPath, fileName);
+ File logFile = new File(logPath.toString());
+ final FileInputStream fis = new FileInputStream(logFile);
+
+ try {
+ LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
+ Set<String> logsSet = new HashSet<String>();
+ logsSet.add(fileName);
+ doReturn(logsSet).when(cli).getMatchedContainerLogFiles(
+ any(ContainerLogsRequest.class), anyBoolean());
+ ClientResponse mockReponse = mock(ClientResponse.class);
+ doReturn(Status.OK).when(mockReponse).getStatusInfo();
+ doReturn(fis).when(mockReponse).getEntityInputStream();
+ doReturn(mockReponse).when(cli).getResponeFromNMWebService(
+ any(Configuration.class),
+ any(Client.class),
+ any(ContainerLogsRequest.class), anyString());
+ cli.setConf(new YarnConfiguration());
+ int exitCode = cli.run(new String[] {"-containerId",
+ containerId1.toString()});
+ assertTrue(exitCode == 0);
+ assertTrue(sysOutStream.toString().contains(
+ logMessage(containerId1, "syslog")));
+ sysOutStream.reset();
+ } finally {
+ IOUtils.closeQuietly(fis);
+ fs.delete(new Path(rootLogDir), true);
+ }
+ }
+
+ @Test (timeout = 5000)
public void testFetchRunningApplicationLogs() throws Exception {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HADOOP-13333. testConf.xml ls
comparators in wrong order (Vrushali C via Varun Saxena)
Posted by cn...@apache.org.
HADOOP-13333. testConf.xml ls comparators in wrong order (Vrushali C via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d714030b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d714030b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d714030b
Branch: refs/heads/HADOOP-13345
Commit: d714030b5d4124f307c09d716d72a9f5a4a25995
Parents: bed69d1
Author: Varun Saxena <va...@apache.org>
Authored: Tue Aug 16 03:03:44 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Tue Aug 16 03:03:44 2016 +0530
----------------------------------------------------------------------
.../hadoop-common/src/test/resources/testConf.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d714030b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index bbbc1ec..82bc789 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -106,11 +106,11 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*-q\s+Print \? instead of non-printable characters\.( )*</expected-output>
+ <expected-output>^\s*rather than a number of bytes\.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*rather than a number of bytes\.( )*</expected-output>
+ <expected-output>^\s*-q\s+Print \? instead of non-printable characters\.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDFS-10747.
o.a.h.hdfs.tools.DebugAdmin usage message is misleading. (Contributed by
Mingliang Liu)
Posted by cn...@apache.org.
HDFS-10747. o.a.h.hdfs.tools.DebugAdmin usage message is misleading. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef55fe17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef55fe17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef55fe17
Branch: refs/heads/HADOOP-13345
Commit: ef55fe171691446d38e6a14e92c1fd4d3d0c64c5
Parents: 382d615
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:23:47 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:23:47 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 4 ++--
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef55fe17/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index d179a5c..a2b91ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -86,7 +86,7 @@ public class DebugAdmin extends Configured implements Tool {
private class VerifyBlockChecksumCommand extends DebugCommand {
VerifyBlockChecksumCommand() {
super("verify",
-"verify [-meta <metadata-file>] [-block <block-file>]",
+"verify -meta <metadata-file> [-block <block-file>]",
" Verify HDFS metadata and block files. If a block file is specified, we\n" +
" will verify that the checksums in the metadata file match the block\n" +
" file.");
@@ -200,7 +200,7 @@ public class DebugAdmin extends Configured implements Tool {
private class RecoverLeaseCommand extends DebugCommand {
RecoverLeaseCommand() {
super("recoverLease",
-"recoverLease [-path <path>] [-retries <num-retries>]",
+"recoverLease -path <path> [-retries <num-retries>]",
" Recover the lease on the specified path. The path must reside on an\n" +
" HDFS filesystem. The default number of retries is 1.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef55fe17/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index cbc293f..22886d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -604,7 +604,7 @@ Useful commands to help administrators debug HDFS issues, like validating block
### `verify`
-Usage: `hdfs debug verify [-meta <metadata-file>] [-block <block-file>]`
+Usage: `hdfs debug verify -meta <metadata-file> [-block <block-file>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -615,7 +615,7 @@ Verify HDFS metadata and block files. If a block file is specified, we will veri
### `recoverLease`
-Usage: `hdfs debug recoverLease [-path <path>] [-retries <num-retries>]`
+Usage: `hdfs debug recoverLease -path <path> [-retries <num-retries>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HADOOP-13208. S3A
listFiles(recursive=true) to do a bulk listObjects instead of walking the
pseudo-tree of directories. Contributed by Steve Loughran.
Posted by cn...@apache.org.
HADOOP-13208. S3A listFiles(recursive=true) to do a bulk listObjects instead of walking the pseudo-tree of directories. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/822d661b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/822d661b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/822d661b
Branch: refs/heads/HADOOP-13345
Commit: 822d661b8fcc42bec6eea958d9fd02ef1aaa4b6c
Parents: 8693936
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed Aug 17 14:54:54 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Wed Aug 17 14:54:54 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/Listing.java | 594 +++++++++++++++++++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 372 +++++++-----
.../hadoop/fs/s3a/S3AInstrumentation.java | 1 +
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 12 +
.../org/apache/hadoop/fs/s3a/Statistic.java | 2 +
.../s3a/TestS3AContractGetFileStatus.java | 16 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 8 +-
.../s3a/scale/TestS3ADirectoryPerformance.java | 58 +-
8 files changed, 897 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
new file mode 100644
index 0000000..4120b20
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import static org.apache.hadoop.fs.s3a.Constants.S3N_FOLDER_SUFFIX;
+import static org.apache.hadoop.fs.s3a.S3AUtils.createFileStatus;
+import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory;
+import static org.apache.hadoop.fs.s3a.S3AUtils.stringify;
+import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
+
+/**
+ * Place for the S3A listing classes; keeps all the small classes under control.
+ */
+public class Listing {
+
+ private final S3AFileSystem owner;
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ public Listing(S3AFileSystem owner) {
+ this.owner = owner;
+ }
+
+ /**
+ * Create a FileStatus iterator against a path, with a given
+ * list object request.
+ * @param listPath path of the listing
+ * @param request initial request to make
+ * @param filter the filter on which paths to accept
+ * @param acceptor the class/predicate to decide which entries to accept
+ * in the listing based on the full file status.
+ * @return the iterator
+ * @throws IOException IO Problems
+ */
+ FileStatusListingIterator createFileStatusListingIterator(
+ Path listPath,
+ ListObjectsRequest request,
+ PathFilter filter,
+ Listing.FileStatusAcceptor acceptor) throws IOException {
+ return new FileStatusListingIterator(
+ new ObjectListingIterator(listPath, request),
+ filter,
+ acceptor);
+ }
+
+ /**
+ * Create a located status iterator over a file status iterator.
+ * @param statusIterator an iterator over the remote status entries
+ * @return a new remote iterator
+ */
+ LocatedFileStatusIterator createLocatedFileStatusIterator(
+ RemoteIterator<FileStatus> statusIterator) {
+ return new LocatedFileStatusIterator(statusIterator);
+ }
+
+ /**
+ * Interface to implement by the logic deciding whether to accept a summary
+ * entry or path as a valid file or directory.
+ */
+ interface FileStatusAcceptor {
+
+ /**
+ * Predicate to decide whether or not to accept a summary entry.
+ * @param keyPath qualified path to the entry
+ * @param summary summary entry
+ * @return true if the entry is accepted (i.e. that a status entry
+ * should be generated.
+ */
+ boolean accept(Path keyPath, S3ObjectSummary summary);
+
+ /**
+ * Predicate to decide whether or not to accept a prefix.
+ * @param keyPath qualified path to the entry
+ * @param commonPrefix the prefix
+ * @return true if the entry is accepted (i.e. that a status entry
+ * should be generated.)
+ */
+ boolean accept(Path keyPath, String commonPrefix);
+ }
+
+ /**
+ * A remote iterator which only iterates over a single `LocatedFileStatus`
+ * value.
+ *
+ * If the status value is null, the iterator declares that it has no data.
+ * This iterator is used to handle {@link listStatus()} calls where the path
+ * handed in refers to a file, not a directory: this is the iterator
+ * returned.
+ */
+ static final class SingleStatusRemoteIterator
+ implements RemoteIterator<LocatedFileStatus> {
+
+ /**
+ * The status to return; set to null after the first iteration.
+ */
+ private LocatedFileStatus status;
+
+ /**
+ * Constructor.
+ * @param status status value: may be null, in which case
+ * the iterator is empty.
+ */
+ public SingleStatusRemoteIterator(LocatedFileStatus status) {
+ this.status = status;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @return true if there is a file status to return: this is always false
+ * for the second iteration, and may be false for the first.
+ * @throws IOException never
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ return status != null;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @return the non-null status element passed in when the instance was
+ * constructed, if it ha not already been retrieved.
+ * @throws IOException never
+ * @throws NoSuchElementException if this is the second call, or it is
+ * the first call and a null {@link LocatedFileStatus} entry was passed
+ * to the constructor.
+ */
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ if (hasNext()) {
+ LocatedFileStatus s = this.status;
+ status = null;
+ return s;
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+ }
+
+ /**
+ * Wraps up object listing into a remote iterator which will ask for more
+ * listing data if needed.
+ *
+ * This is a complex operation, especially the process to determine
+ * if there are more entries remaining. If there are no more results
+ * remaining in the (filtered) results of the current listing request, then
+ * another request is made <i>and those results filtered</i> before the
+ * iterator can declare that there is more data available.
+ *
+ * The need to filter the results precludes the iterator from simply
+ * declaring that if the {@link S3AFileSystem.ObjectListingIterator#hasNext()}
+ * is true then there are more results. Instead the next batch of results must
+ * be retrieved and filtered.
+ *
+ * What does this mean? It means that remote requests to retrieve new
+ * batches of object listings are made in the {@link #hasNext()} call;
+ * the {@link #next()} call simply returns the filtered results of the last
+ * listing processed. However, do note that {@link #next()} calls
+ * {@link #hasNext()} during its operation. This is critical to ensure
+ * that a listing obtained through a sequence of {@link #next()} will
+ * complete with the same set of results as a classic
+ * {@code while(it.hasNext()} loop.
+ *
+ * Thread safety: None.
+ */
+ class FileStatusListingIterator
+ implements RemoteIterator<FileStatus> {
+
+ /** Source of objects. */
+ private final ObjectListingIterator source;
+ /** Filter of paths from API call. */
+ private final PathFilter filter;
+ /** Filter of entries from file status. */
+ private final FileStatusAcceptor acceptor;
+ /** request batch size. */
+ private int batchSize;
+ /** Iterator over the current set of results. */
+ private ListIterator<FileStatus> statusBatchIterator;
+
+ /**
+ * Create an iterator over file status entries.
+ * @param source the listing iterator from a listObjects call.
+ * @param filter the filter on which paths to accept
+ * @param acceptor the class/predicate to decide which entries to accept
+ * in the listing based on the full file status.
+ * @throws IOException IO Problems
+ */
+ FileStatusListingIterator(ObjectListingIterator source,
+ PathFilter filter,
+ FileStatusAcceptor acceptor) throws IOException {
+ this.source = source;
+ this.filter = filter;
+ this.acceptor = acceptor;
+ // build the first set of results. This will not trigger any
+ // remote IO, assuming the source iterator is in its initial
+ // iteration
+ requestNextBatch();
+ }
+
+ /**
+ * Report whether or not there is new data available.
+ * If there is data in the local filtered list, return true.
+ * Else: request more data util that condition is met, or there
+ * is no more remote listing data.
+ * @return true if a call to {@link #next()} will succeed.
+ * @throws IOException
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ return statusBatchIterator.hasNext() || requestNextBatch();
+ }
+
+ @Override
+ public FileStatus next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return statusBatchIterator.next();
+ }
+
+ /**
+ * Try to retrieve another batch.
+ * Note that for the initial batch,
+ * {@link S3AFileSystem.ObjectListingIterator} does not generate a request;
+ * it simply returns the initial set.
+ *
+ * @return true if a new batch was created.
+ * @throws IOException IO problems
+ */
+ private boolean requestNextBatch() throws IOException {
+ // look for more object listing batches being available
+ while (source.hasNext()) {
+ // if available, retrieve it and build the next status
+ if (buildNextStatusBatch(source.next())) {
+ // this batch successfully generated entries matching the filters/
+ // acceptors; declare that the request was successful
+ return true;
+ } else {
+ LOG.debug("All entries in batch were filtered...continuing");
+ }
+ }
+ // if this code is reached, it means that all remaining
+ // object lists have been retrieved, and there are no new entries
+ // to return.
+ return false;
+ }
+
+ /**
+ * Build the next status batch from a listing.
+ * @param objects the next object listing
+ * @return true if this added any entries after filtering
+ */
+ private boolean buildNextStatusBatch(ObjectListing objects) {
+ // counters for debug logs
+ int added = 0, ignored = 0;
+ // list to fill in with results. Initial size will be list maximum.
+ List<FileStatus> stats = new ArrayList<>(
+ objects.getObjectSummaries().size() +
+ objects.getCommonPrefixes().size());
+ // objects
+ for (S3ObjectSummary summary : objects.getObjectSummaries()) {
+ String key = summary.getKey();
+ Path keyPath = owner.keyToQualifiedPath(key);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: {}", keyPath, stringify(summary));
+ }
+ // Skip over keys that are ourselves and old S3N _$folder$ files
+ if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
+ FileStatus status = createFileStatus(keyPath, summary,
+ owner.getDefaultBlockSize(keyPath));
+ LOG.debug("Adding: {}", status);
+ stats.add(status);
+ added++;
+ } else {
+ LOG.debug("Ignoring: {}", keyPath);
+ ignored++;
+ }
+ }
+
+ // prefixes: always directories
+ for (String prefix : objects.getCommonPrefixes()) {
+ Path keyPath = owner.keyToQualifiedPath(prefix);
+ if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
+ FileStatus status = new S3AFileStatus(true, false, keyPath);
+ LOG.debug("Adding directory: {}", status);
+ added++;
+ stats.add(status);
+ } else {
+ LOG.debug("Ignoring directory: {}", keyPath);
+ ignored++;
+ }
+ }
+
+ // finish up
+ batchSize = stats.size();
+ statusBatchIterator = stats.listIterator();
+ boolean hasNext = statusBatchIterator.hasNext();
+ LOG.debug("Added {} entries; ignored {}; hasNext={}; hasMoreObjects={}",
+ added, ignored, hasNext, objects.isTruncated());
+ return hasNext;
+ }
+
+ /**
+ * Get the number of entries in the current batch.
+ * @return a number, possibly zero.
+ */
+ public int getBatchSize() {
+ return batchSize;
+ }
+ }
+
+ /**
+ * Wraps up AWS `ListObjects` requests in a remote iterator
+ * which will ask for more listing data if needed.
+ *
+ * That is:
+ *
+ * 1. The first invocation of the {@link #next()} call will return the results
+ * of the first request, the one created during the construction of the
+ * instance.
+ *
+ * 2. Second and later invocations will continue the ongoing listing,
+ * calling {@link #continueListObjects(ObjectListing)} to request the next
+ * batch of results.
+ *
+ * 3. The {@link #hasNext()} predicate returns true for the initial call,
+ * where {@link #next()} will return the initial results. It declares
+ * that it has future results iff the last executed request was truncated.
+ *
+ * Thread safety: none.
+ */
+ class ObjectListingIterator implements RemoteIterator<ObjectListing> {
+
+ /** The path listed. */
+ private final Path listPath;
+
+ /** The most recent listing results. */
+ private ObjectListing objects;
+
+ /** Indicator that this is the first listing. */
+ private boolean firstListing = true;
+
+ /**
+ * Count of how many listings have been requested
+ * (including initial result).
+ */
+ private int listingCount = 1;
+
+ /**
+ * Maximum keys in a request.
+ */
+ private int maxKeys;
+
+ /**
+ * Constructor -calls `listObjects()` on the request to populate the
+ * initial set of results/fail if there was a problem talking to the bucket.
+ * @param listPath path of the listing
+ * @param request initial request to make
+ * */
+ ObjectListingIterator(
+ Path listPath,
+ ListObjectsRequest request) {
+ this.listPath = listPath;
+ this.maxKeys = owner.getMaxKeys();
+ this.objects = owner.listObjects(request);
+ }
+
+ /**
+ * Declare that the iterator has data if it is either is the initial
+ * iteration or it is a later one and the last listing obtained was
+ * incomplete.
+ * @throws IOException never: there is no IO in this operation.
+ */
+ @Override
+ public boolean hasNext() throws IOException {
+ return firstListing || objects.isTruncated();
+ }
+
+ /**
+ * Ask for the next listing.
+ * For the first invocation, this returns the initial set, with no
+ * remote IO. For later requests, S3 will be queried, hence the calls
+ * may block or fail.
+ * @return the next object listing.
+ * @throws IOException if a query made of S3 fails.
+ * @throws NoSuchElementException if there is no more data to list.
+ */
+ @Override
+ public ObjectListing next() throws IOException {
+ if (firstListing) {
+ // on the first listing, don't request more data.
+ // Instead just clear the firstListing flag so that it future calls
+ // will request new data.
+ firstListing = false;
+ } else {
+ try {
+ if (!objects.isTruncated()) {
+ // nothing more to request: fail.
+ throw new NoSuchElementException("No more results in listing of "
+ + listPath);
+ }
+ // need to request a new set of objects.
+ LOG.debug("[{}], Requesting next {} objects under {}",
+ listingCount, maxKeys, listPath);
+ objects = owner.continueListObjects(objects);
+ listingCount++;
+ LOG.debug("New listing status: {}", this);
+ } catch (AmazonClientException e) {
+ throw translateException("listObjects()", listPath, e);
+ }
+ }
+ return objects;
+ }
+
+ @Override
+ public String toString() {
+ return "Object listing iterator against " + listPath
+ + "; listing count "+ listingCount
+ + "; isTruncated=" + objects.isTruncated();
+ }
+
+ /**
+ * Get the path listed.
+ * @return the path used in this listing.
+ */
+ public Path getListPath() {
+ return listPath;
+ }
+
+ /**
+ * Get the count of listing requests.
+ * @return the counter of requests made (including the initial lookup).
+ */
+ public int getListingCount() {
+ return listingCount;
+ }
+ }
+
+ /**
+ * Accept all entries except the base path and those which map to S3N
+ * pseudo directory markers.
+ */
+ static class AcceptFilesOnly implements FileStatusAcceptor {
+ private final Path qualifiedPath;
+
+ public AcceptFilesOnly(Path qualifiedPath) {
+ this.qualifiedPath = qualifiedPath;
+ }
+
+ /**
+ * Reject a summary entry if the key path is the qualified Path, or
+ * it ends with {@code "_$folder$"}.
+ * @param keyPath key path of the entry
+ * @param summary summary entry
+ * @return true if the entry is accepted (i.e. that a status entry
+ * should be generated.
+ */
+ @Override
+ public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ return !keyPath.equals(qualifiedPath)
+ && !summary.getKey().endsWith(S3N_FOLDER_SUFFIX)
+ && !objectRepresentsDirectory(summary.getKey(), summary.getSize());
+ }
+
+ /**
+ * Accept no directory paths.
+ * @param keyPath qualified path to the entry
+ * @param prefix common prefix in listing.
+ * @return false, always.
+ */
+ @Override
+ public boolean accept(Path keyPath, String prefix) {
+ return false;
+ }
+ }
+
+ /**
+ * Take a remote iterator over a set of {@link FileStatus} instances and
+ * return a remote iterator of {@link LocatedFileStatus} instances.
+ */
+ class LocatedFileStatusIterator
+ implements RemoteIterator<LocatedFileStatus> {
+ private final RemoteIterator<FileStatus> statusIterator;
+
+ /**
+ * Constructor.
+ * @param statusIterator an iterator over the remote status entries
+ */
+ LocatedFileStatusIterator(RemoteIterator<FileStatus> statusIterator) {
+ this.statusIterator = statusIterator;
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ return statusIterator.hasNext();
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ return owner.toLocatedFileStatus(statusIterator.next());
+ }
+ }
+
+ /**
+ * Accept all entries except the base path and those which map to S3N
+ * pseudo directory markers.
+ */
+ static class AcceptAllButSelfAndS3nDirs implements FileStatusAcceptor {
+
+ /** Base path. */
+ private final Path qualifiedPath;
+
+ /**
+ * Constructor.
+ * @param qualifiedPath an already-qualified path.
+ */
+ public AcceptAllButSelfAndS3nDirs(Path qualifiedPath) {
+ this.qualifiedPath = qualifiedPath;
+ }
+
+ /**
+ * Reject a summary entry if the key path is the qualified Path, or
+ * it ends with {@code "_$folder$"}.
+ * @param keyPath key path of the entry
+ * @param summary summary entry
+ * @return true if the entry is accepted (i.e. that a status entry
+ * should be generated.)
+ */
+ @Override
+ public boolean accept(Path keyPath, S3ObjectSummary summary) {
+ return !keyPath.equals(qualifiedPath) &&
+ !summary.getKey().endsWith(S3N_FOLDER_SUFFIX);
+ }
+
+ /**
+ * Accept all prefixes except the one for the base path, "self".
+ * @param keyPath qualified path to the entry
+ * @param prefix common prefix in listing.
+ * @return true if the entry is accepted (i.e. that a status entry
+ * should be generated.
+ */
+ @Override
+ public boolean accept(Path keyPath, String prefix) {
+ return !keyPath.equals(qualifiedPath);
+ }
+ }
+
+ /**
+ * A Path filter which accepts all filenames.
+ */
+ static final PathFilter ACCEPT_ALL = new PathFilter() {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "ACCEPT_ALL";
+ }
+ };
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 902f73a..1cf2631 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.VersionInfo;
import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
@@ -115,6 +116,7 @@ public class S3AFileSystem extends FileSystem {
private AmazonS3Client s3;
private String bucket;
private int maxKeys;
+ private Listing listing;
private long partSize;
private boolean enableMultiObjectsDelete;
private TransferManager transfers;
@@ -181,6 +183,7 @@ public class S3AFileSystem extends FileSystem {
initAmazonS3Client(conf, credentials, awsConf);
maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
+ listing = new Listing(this);
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
if (partSize < 5 * 1024 * 1024) {
LOG.error(MULTIPART_SIZE + " must be at least 5 MB");
@@ -514,7 +517,11 @@ public class S3AFileSystem extends FileSystem {
super();
}
- /* Turns a path (relative or otherwise) into an S3 key
+ /**
+ * Turns a path (relative or otherwise) into an S3 key.
+ *
+ * @param path input path, may be relative to the working dir
+ * @return a key excluding the leading "/", or, if it is the root path, ""
*/
private String pathToKey(Path path) {
if (!path.isAbsolute()) {
@@ -528,11 +535,50 @@ public class S3AFileSystem extends FileSystem {
return path.toUri().getPath().substring(1);
}
+ /**
+ * Turns a path (relative or otherwise) into an S3 key, adding a trailing
+ * "/" if the path is not the root <i>and</i> does not already have a "/"
+ * at the end.
+ *
+ * @param key s3 key or ""
+ * @return the with a trailing "/", or, if it is the root key, "",
+ */
+ private String maybeAddTrailingSlash(String key) {
+ if (!key.isEmpty() && !key.endsWith("/")) {
+ return key + '/';
+ } else {
+ return key;
+ }
+ }
+
+ /**
+ * Convert a path back to a key.
+ * @param key input key
+ * @return the path from this key
+ */
private Path keyToPath(String key) {
return new Path("/" + key);
}
/**
+ * Convert a key to a fully qualified path.
+ * @param key input key
+ * @return the fully qualified path including URI scheme and bucket name.
+ */
+ Path keyToQualifiedPath(String key) {
+ return qualify(keyToPath(key));
+ }
+
+ /**
+ * Qualify a path.
+ * @param path path to qualify
+ * @return a qualified path.
+ */
+ Path qualify(Path path) {
+ return path.makeQualified(uri, workingDir);
+ }
+
+ /**
* Check that a Path belongs to this FileSystem.
* Unlike the superclass, this version does not look at authority,
* only hostnames.
@@ -550,10 +596,10 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * Opens an FSDataInputStream at the indicated Path.
- * @param f the file name to open
- * @param bufferSize the size of the buffer to be used.
- */
+ * Opens an FSDataInputStream at the indicated Path.
+ * @param f the file name to open
+ * @param bufferSize the size of the buffer to be used.
+ */
public FSDataInputStream open(Path f, int bufferSize)
throws IOException {
@@ -875,7 +921,7 @@ public class S3AFileSystem extends FileSystem {
* @return the next result object
*/
protected ObjectListing continueListObjects(ObjectListing objects) {
- incrementStatistic(OBJECT_LIST_REQUESTS);
+ incrementStatistic(OBJECT_CONTINUE_LIST_REQUESTS);
incrementReadOperations();
return s3.listNextBatchOfObjects(objects);
}
@@ -1132,12 +1178,7 @@ public class S3AFileSystem extends FileSystem {
} else {
LOG.debug("Getting objects for directory prefix {} to delete", key);
- ListObjectsRequest request = new ListObjectsRequest();
- request.setBucketName(bucket);
- request.setPrefix(key);
- // Hopefully not setting a delimiter will cause this to find everything
- //request.setDelimiter("/");
- request.setMaxKeys(maxKeys);
+ ListObjectsRequest request = createListObjectsRequest(key, null);
ObjectListing objects = listObjects(request);
List<DeleteObjectsRequest.KeyVersion> keys =
@@ -1211,66 +1252,57 @@ public class S3AFileSystem extends FileSystem {
*/
public FileStatus[] innerListStatus(Path f) throws FileNotFoundException,
IOException, AmazonClientException {
- String key = pathToKey(f);
- LOG.debug("List status for path: {}", f);
+ Path path = qualify(f);
+ String key = pathToKey(path);
+ LOG.debug("List status for path: {}", path);
incrementStatistic(INVOCATION_LIST_STATUS);
- final List<FileStatus> result = new ArrayList<FileStatus>();
- final FileStatus fileStatus = getFileStatus(f);
+ List<FileStatus> result;
+ final FileStatus fileStatus = getFileStatus(path);
if (fileStatus.isDirectory()) {
if (!key.isEmpty()) {
- key = key + "/";
+ key = key + '/';
}
- ListObjectsRequest request = new ListObjectsRequest();
- request.setBucketName(bucket);
- request.setPrefix(key);
- request.setDelimiter("/");
- request.setMaxKeys(maxKeys);
-
+ ListObjectsRequest request = createListObjectsRequest(key, "/");
LOG.debug("listStatus: doing listObjects for directory {}", key);
- ObjectListing objects = listObjects(request);
-
- Path fQualified = f.makeQualified(uri, workingDir);
-
- while (true) {
- for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
- // Skip over keys that are ourselves and old S3N _$folder$ files
- if (keyPath.equals(fQualified) ||
- summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
- LOG.debug("Ignoring: {}", keyPath);
- } else {
- S3AFileStatus status = createFileStatus(keyPath, summary,
- getDefaultBlockSize(keyPath));
- result.add(status);
- LOG.debug("Adding: {}", status);
- }
- }
-
- for (String prefix : objects.getCommonPrefixes()) {
- Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
- if (!keyPath.equals(f)) {
- result.add(new S3AFileStatus(true, false, keyPath));
- LOG.debug("Adding: rd: {}", keyPath);
- }
- }
-
- if (objects.isTruncated()) {
- LOG.debug("listStatus: list truncated - getting next batch");
- objects = continueListObjects(objects);
- } else {
- break;
- }
+ Listing.FileStatusListingIterator files =
+ listing.createFileStatusListingIterator(path,
+ request,
+ ACCEPT_ALL,
+ new Listing.AcceptAllButSelfAndS3nDirs(path));
+ result = new ArrayList<>(files.getBatchSize());
+ while (files.hasNext()) {
+ result.add(files.next());
}
+ return result.toArray(new FileStatus[result.size()]);
} else {
- LOG.debug("Adding: rd (not a dir): {}", f);
- result.add(fileStatus);
+ LOG.debug("Adding: rd (not a dir): {}", path);
+ FileStatus[] stats = new FileStatus[1];
+ stats[0]= fileStatus;
+ return stats;
}
+ }
- return result.toArray(new FileStatus[result.size()]);
+ /**
+ * Create a {@code ListObjectsRequest} request against this bucket,
+ * with the maximum keys returned in a query set by {@link #maxKeys}.
+ * @param key key for request
+ * @param delimiter any delimiter
+ * @return the request
+ */
+ private ListObjectsRequest createListObjectsRequest(String key,
+ String delimiter) {
+ ListObjectsRequest request = new ListObjectsRequest();
+ request.setBucketName(bucket);
+ request.setMaxKeys(maxKeys);
+ request.setPrefix(key);
+ if (delimiter != null) {
+ request.setDelimiter(delimiter);
+ }
+ return request;
}
/**
@@ -1371,11 +1403,11 @@ public class S3AFileSystem extends FileSystem {
* @throws java.io.FileNotFoundException when the path does not exist;
* @throws IOException on other problems.
*/
- public S3AFileStatus getFileStatus(Path f) throws IOException {
- String key = pathToKey(f);
+ public S3AFileStatus getFileStatus(final Path f) throws IOException {
incrementStatistic(INVOCATION_GET_FILE_STATUS);
- LOG.debug("Getting path status for {} ({})", f , key);
-
+ final Path path = qualify(f);
+ String key = pathToKey(path);
+ LOG.debug("Getting path status for {} ({})", path , key);
if (!key.isEmpty()) {
try {
ObjectMetadata meta = getObjectMetadata(key);
@@ -1383,20 +1415,20 @@ public class S3AFileSystem extends FileSystem {
if (objectRepresentsDirectory(key, meta.getContentLength())) {
LOG.debug("Found exact file: fake directory");
return new S3AFileStatus(true, true,
- f.makeQualified(uri, workingDir));
+ path);
} else {
LOG.debug("Found exact file: normal file");
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
- f.makeQualified(uri, workingDir),
- getDefaultBlockSize(f.makeQualified(uri, workingDir)));
+ path,
+ getDefaultBlockSize(path));
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
- throw translateException("getFileStatus", f, e);
+ throw translateException("getFileStatus", path, e);
}
} catch (AmazonClientException e) {
- throw translateException("getFileStatus", f, e);
+ throw translateException("getFileStatus", path, e);
}
// Necessary?
@@ -1407,14 +1439,14 @@ public class S3AFileSystem extends FileSystem {
if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
LOG.debug("Found file (with /): fake directory");
- return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
+ return new S3AFileStatus(true, true, path);
} else {
LOG.warn("Found file (with /): real file? should not happen: {}", key);
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
- f.makeQualified(uri, workingDir),
- getDefaultBlockSize(f.makeQualified(uri, workingDir)));
+ path,
+ getDefaultBlockSize(path));
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
@@ -1427,9 +1459,7 @@ public class S3AFileSystem extends FileSystem {
}
try {
- if (!key.isEmpty() && !key.endsWith("/")) {
- key = key + "/";
- }
+ key = maybeAddTrailingSlash(key);
ListObjectsRequest request = new ListObjectsRequest();
request.setBucketName(bucket);
request.setPrefix(key);
@@ -1453,11 +1483,10 @@ public class S3AFileSystem extends FileSystem {
}
}
- return new S3AFileStatus(true, false,
- f.makeQualified(uri, workingDir));
+ return new S3AFileStatus(true, false, path);
} else if (key.isEmpty()) {
LOG.debug("Found root directory");
- return new S3AFileStatus(true, true, f.makeQualified(uri, workingDir));
+ return new S3AFileStatus(true, true, path);
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
@@ -1467,8 +1496,8 @@ public class S3AFileSystem extends FileSystem {
throw translateException("getFileStatus", key, e);
}
- LOG.debug("Not Found: {}", f);
- throw new FileNotFoundException("No such file or directory: " + f);
+ LOG.debug("Not Found: {}", path);
+ throw new FileNotFoundException("No such file or directory: " + path);
}
/**
@@ -1829,7 +1858,15 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * Override superclass so as to add statistic collection.
+ * Get the maximum key count.
+ * @return a value, valid after initialization
+ */
+ int getMaxKeys() {
+ return maxKeys;
+ }
+
+ /**
+ * Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* {@inheritDoc}
*/
@Override
@@ -1854,24 +1891,6 @@ public class S3AFileSystem extends FileSystem {
* {@inheritDoc}
*/
@Override
- public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
- throws FileNotFoundException, IOException {
- incrementStatistic(INVOCATION_LIST_LOCATED_STATUS);
- return super.listLocatedStatus(f);
- }
-
- @Override
- public RemoteIterator<LocatedFileStatus> listFiles(Path f,
- boolean recursive) throws FileNotFoundException, IOException {
- incrementStatistic(INVOCATION_LIST_FILES);
- return super.listFiles(f, recursive);
- }
-
- /**
- * Override superclass so as to add statistic collection.
- * {@inheritDoc}
- */
- @Override
public boolean exists(Path f) throws IOException {
incrementStatistic(INVOCATION_EXISTS);
return super.exists(f);
@@ -1898,6 +1917,129 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * {@inheritDoc}.
+ *
+ * This implementation is optimized for S3, which can do a bulk listing
+ * off all entries under a path in one single operation. Thus there is
+ * no need to recursively walk the directory tree.
+ *
+ * Instead a {@link ListObjectsRequest} is created requesting a (windowed)
+ * listing of all entries under the given path. This is used to construct
+ * an {@code ObjectListingIterator} instance, iteratively returning the
+ * sequence of lists of elements under the path. This is then iterated
+ * over in a {@code FileStatusListingIterator}, which generates
+ * {@link S3AFileStatus} instances, one per listing entry.
+ * These are then translated into {@link LocatedFileStatus} instances.
+ *
+ * This is essentially a nested and wrapped set of iterators, with some
+ * generator classes; an architecture which may become less convoluted
+ * using lambda-expressions.
+ * @param f a path
+ * @param recursive if the subdirectories need to be traversed recursively
+ *
+ * @return an iterator that traverses statuses of the files/directories
+ * in the given path
+ * @throws FileNotFoundException if {@code path} does not exist
+ * @throws IOException if any I/O error occurred
+ */
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(Path f,
+ boolean recursive) throws FileNotFoundException, IOException {
+ incrementStatistic(INVOCATION_LIST_FILES);
+ Path path = qualify(f);
+ LOG.debug("listFiles({}, {})", path, recursive);
+ try {
+ // lookup dir triggers existence check
+ final FileStatus fileStatus = getFileStatus(path);
+ if (fileStatus.isFile()) {
+ // simple case: File
+ LOG.debug("Path is a file");
+ return new Listing.SingleStatusRemoteIterator(
+ toLocatedFileStatus(fileStatus));
+ } else {
+ // directory: do a bulk operation
+ String key = maybeAddTrailingSlash(pathToKey(path));
+ String delimiter = recursive ? null : "/";
+ LOG.debug("Requesting all entries under {} with delimiter '{}'",
+ key, delimiter);
+ return listing.createLocatedFileStatusIterator(
+ listing.createFileStatusListingIterator(path,
+ createListObjectsRequest(key, delimiter),
+ ACCEPT_ALL,
+ new Listing.AcceptFilesOnly(path)));
+ }
+ } catch (AmazonClientException e) {
+ throw translateException("listFiles", path, e);
+ }
+ }
+
+ /**
+ * Override superclass so as to add statistic collection.
+ * {@inheritDoc}
+ */
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
+ throws FileNotFoundException, IOException {
+ return listLocatedStatus(f, ACCEPT_ALL);
+ }
+
+ /**
+ * {@inheritDoc}.
+ *
+ * S3 Optimized directory listing. The initial operation performs the
+ * first bulk listing; extra listings will take place
+ * when all the current set of results are used up.
+ * @param f a path
+ * @param filter a path filter
+ * @return an iterator that traverses statuses of the files/directories
+ * in the given path
+ * @throws FileNotFoundException if {@code path} does not exist
+ * @throws IOException if any I/O error occurred
+ */
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
+ final PathFilter filter)
+ throws FileNotFoundException, IOException {
+ incrementStatistic(INVOCATION_LIST_LOCATED_STATUS);
+ Path path = qualify(f);
+ LOG.debug("listLocatedStatus({}, {}", path, filter);
+ try {
+ // lookup dir triggers existence check
+ final FileStatus fileStatus = getFileStatus(path);
+ if (fileStatus.isFile()) {
+ // simple case: File
+ LOG.debug("Path is a file");
+ return new Listing.SingleStatusRemoteIterator(
+ filter.accept(path) ? toLocatedFileStatus(fileStatus) : null);
+ } else {
+ // directory: trigger a lookup
+ String key = maybeAddTrailingSlash(pathToKey(path));
+ return listing.createLocatedFileStatusIterator(
+ listing.createFileStatusListingIterator(path,
+ createListObjectsRequest(key, "/"),
+ filter,
+ new Listing.AcceptAllButSelfAndS3nDirs(path)));
+ }
+ } catch (AmazonClientException e) {
+ throw translateException("listLocatedStatus", path, e);
+ }
+ }
+
+ /**
+ * Build a {@link LocatedFileStatus} from a {@link FileStatus} instance.
+ * @param status file status
+ * @return a located status with block locations set up from this FS.
+ * @throws IOException IO Problems.
+ */
+ LocatedFileStatus toLocatedFileStatus(FileStatus status)
+ throws IOException {
+ return new LocatedFileStatus(status,
+ status.isFile() ?
+ getFileBlockLocations(status, 0, status.getLen())
+ : null);
+ }
+
+ /**
* Get a integer option >= the minimum allowed value.
* @param conf configuration
* @param key key to look up
@@ -1934,38 +2076,4 @@ public class S3AFileSystem extends FileSystem {
return v;
}
- /**
- * This is a simple encapsulation of the
- * S3 access key and secret.
- */
- static class AWSAccessKeys {
- private String accessKey = null;
- private String accessSecret = null;
-
- /**
- * Constructor.
- * @param key - AWS access key
- * @param secret - AWS secret key
- */
- public AWSAccessKeys(String key, String secret) {
- accessKey = key;
- accessSecret = secret;
- }
-
- /**
- * Return the AWS access key.
- * @return key
- */
- public String getAccessKey() {
- return accessKey;
- }
-
- /**
- * Return the AWS secret key.
- * @return secret
- */
- public String getAccessSecret() {
- return accessSecret;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index 514b974..b4c4063 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -95,6 +95,7 @@ public class S3AInstrumentation {
OBJECT_COPY_REQUESTS,
OBJECT_DELETE_REQUESTS,
OBJECT_LIST_REQUESTS,
+ OBJECT_CONTINUE_LIST_REQUESTS,
OBJECT_METADATA_REQUESTS,
OBJECT_MULTIPART_UPLOAD_ABORTED,
OBJECT_PUT_BYTES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 8033ac3..3d48fcd 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -280,4 +280,16 @@ public final class S3AUtils {
return val;
}
}
+
+ /**
+ * String information about a summary entry for debug messages.
+ * @param summary summary object
+ * @return string value
+ */
+ public static String stringify(S3ObjectSummary summary) {
+ StringBuilder builder = new StringBuilder(summary.getKey().length() + 100);
+ builder.append(summary.getKey()).append(' ');
+ builder.append("size=").append(summary.getSize());
+ return builder.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index 36d163c..cbc34d6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -69,6 +69,8 @@ public enum Statistic {
OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),
OBJECT_LIST_REQUESTS("object_list_requests",
"Number of object listings made"),
+ OBJECT_CONTINUE_LIST_REQUESTS("object_continue_list_requests",
+ "Number of continued object listings made"),
OBJECT_METADATA_REQUESTS("object_metadata_requests",
"Number of requests for object metadata"),
OBJECT_MULTIPART_UPLOAD_ABORTED("object_multipart_aborted",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
index d7b8fe3..5937d49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractGetFileStatus.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.contract.s3a;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusTest {
@@ -28,4 +30,18 @@ public class TestS3AContractGetFileStatus extends AbstractContractGetFileStatusT
return new S3AContract(conf);
}
+ @Override
+ public void teardown() throws Exception {
+ getLog().info("FS details {}", getFileSystem());
+ super.teardown();
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ // aggressively low page size forces tests to go multipage
+ conf.setInt(Constants.MAX_PAGING_KEYS, 2);
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 39c6028..e45db48 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -124,7 +124,7 @@ public class S3ATestUtils {
try {
callback.call();
return;
- } catch (FailFastException e) {
+ } catch (InterruptedException | FailFastException e) {
throw e;
} catch (Exception e) {
lastException = e;
@@ -321,7 +321,7 @@ public class S3ATestUtils {
* @return true if the value is {@code ==} the other's
*/
public boolean diffEquals(MetricDiff that) {
- return this.currentValue() == that.currentValue();
+ return this.diff() == that.diff();
}
/**
@@ -330,7 +330,7 @@ public class S3ATestUtils {
* @return true if the value is {@code <} the other's
*/
public boolean diffLessThan(MetricDiff that) {
- return this.currentValue() < that.currentValue();
+ return this.diff() < that.diff();
}
/**
@@ -339,7 +339,7 @@ public class S3ATestUtils {
* @return true if the value is {@code <=} the other's
*/
public boolean diffLessThanOrEquals(MetricDiff that) {
- return this.currentValue() <= that.currentValue();
+ return this.diff() <= that.diff();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/822d661b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
index 7ece394..35ea3ad 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3ADirectoryPerformance.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.s3a.scale;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Statistic;
import org.junit.Test;
@@ -51,6 +50,8 @@ public class TestS3ADirectoryPerformance extends S3AScaleTestBase {
int files = scale;
MetricDiff metadataRequests = new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
MetricDiff listRequests = new MetricDiff(fs, OBJECT_LIST_REQUESTS);
+ MetricDiff listContinueRequests =
+ new MetricDiff(fs, OBJECT_CONTINUE_LIST_REQUESTS);
MetricDiff listStatusCalls = new MetricDiff(fs, INVOCATION_LIST_FILES);
MetricDiff getFileStatusCalls =
new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
@@ -69,34 +70,29 @@ public class TestS3ADirectoryPerformance extends S3AScaleTestBase {
printThenReset(LOG,
metadataRequests,
listRequests,
+ listContinueRequests,
listStatusCalls,
getFileStatusCalls);
+ describe("Listing files via treewalk");
try {
// Scan the directory via an explicit tree walk.
// This is the baseline for any listing speedups.
- MetricDiff treewalkMetadataRequests =
- new MetricDiff(fs, OBJECT_METADATA_REQUESTS);
- MetricDiff treewalkListRequests = new MetricDiff(fs,
- OBJECT_LIST_REQUESTS);
- MetricDiff treewalkListStatusCalls = new MetricDiff(fs,
- INVOCATION_LIST_FILES);
- MetricDiff treewalkGetFileStatusCalls =
- new MetricDiff(fs, INVOCATION_GET_FILE_STATUS);
NanoTimer treeWalkTimer = new NanoTimer();
TreeScanResults treewalkResults = treeWalk(fs, listDir);
- treeWalkTimer.end("List status via treewalk");
+ treeWalkTimer.end("List status via treewalk of %s", created);
- print(LOG,
- treewalkMetadataRequests,
- treewalkListRequests,
- treewalkListStatusCalls,
- treewalkGetFileStatusCalls);
+ printThenReset(LOG,
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
assertEquals("Files found in listFiles(recursive=true) " +
" created=" + created + " listed=" + treewalkResults,
created.getFileCount(), treewalkResults.getFileCount());
-
+ describe("Listing files via listFiles(recursive=true)");
// listFiles() does the recursion internally
NanoTimer listFilesRecursiveTimer = new NanoTimer();
@@ -108,31 +104,33 @@ public class TestS3ADirectoryPerformance extends S3AScaleTestBase {
" created=" + created + " listed=" + listFilesResults,
created.getFileCount(), listFilesResults.getFileCount());
- treewalkListRequests.assertDiffEquals(listRequests);
- printThenReset(LOG,
- metadataRequests, listRequests,
- listStatusCalls, getFileStatusCalls);
-
- NanoTimer globStatusTimer = new NanoTimer();
- FileStatus[] globStatusFiles = fs.globStatus(listDir);
- globStatusTimer.end("Time to globStatus() %s", globStatusTimer);
- LOG.info("Time for glob status {} entries: {}",
- globStatusFiles.length,
- toHuman(createTimer.duration()));
- printThenReset(LOG,
+ // only two list operations should have taken place
+ print(LOG,
metadataRequests,
listRequests,
+ listContinueRequests,
listStatusCalls,
getFileStatusCalls);
+ assertEquals(listRequests.toString(), 2, listRequests.diff());
+ reset(metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
+
} finally {
+ describe("deletion");
// deletion at the end of the run
NanoTimer deleteTimer = new NanoTimer();
fs.delete(listDir, true);
deleteTimer.end("Deleting directory tree");
printThenReset(LOG,
- metadataRequests, listRequests,
- listStatusCalls, getFileStatusCalls);
+ metadataRequests,
+ listRequests,
+ listContinueRequests,
+ listStatusCalls,
+ getFileStatusCalls);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-10641.
TestBlockManager#testBlockReportQueueing fails intermittently. (Contributed
by Daryn Sharp)
Posted by cn...@apache.org.
HDFS-10641. TestBlockManager#testBlockReportQueueing fails intermittently. (Contributed by Daryn Sharp)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d4d95fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d4d95fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d4d95fd
Branch: refs/heads/HADOOP-13345
Commit: 4d4d95fdd5e1e985c16005adc45517cc8b549ae8
Parents: 9336a04
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Aug 15 20:28:40 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Aug 15 20:28:40 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/blockmanagement/TestBlockManager.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d4d95fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 535acd7..1f58f99 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1013,6 +1013,7 @@ public class TestBlockManager {
final CyclicBarrier startBarrier = new CyclicBarrier(2);
final CountDownLatch endLatch = new CountDownLatch(3);
+ final CountDownLatch doneLatch = new CountDownLatch(1);
// create a task intended to block while processing, thus causing
// the queue to backup. simulates how a full BR is processed.
@@ -1020,7 +1021,7 @@ public class TestBlockManager {
new Callable<Void>(){
@Override
public Void call() throws IOException {
- return bm.runBlockOp(new Callable<Void>() {
+ bm.runBlockOp(new Callable<Void>() {
@Override
public Void call()
throws InterruptedException, BrokenBarrierException {
@@ -1030,6 +1031,9 @@ public class TestBlockManager {
return null;
}
});
+ // signal that runBlockOp returned
+ doneLatch.countDown();
+ return null;
}
});
@@ -1074,7 +1078,7 @@ public class TestBlockManager {
startBarrier.await(1, TimeUnit.SECONDS);
assertTrue(endLatch.await(1, TimeUnit.SECONDS));
assertEquals(0, bm.getBlockOpQueueLength());
- assertTrue(blockingOp.isDone());
+ assertTrue(doneLatch.await(1, TimeUnit.SECONDS));
} finally {
cluster.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-13494. ReconfigurableBase
can log sensitive information. Contributed by Sean Mackrory.
Posted by cn...@apache.org.
HADOOP-13494. ReconfigurableBase can log sensitive information. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b689e7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b689e7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b689e7a
Branch: refs/heads/HADOOP-13345
Commit: 4b689e7a758a55cec2ca8398727feefc8ac21bfd
Parents: 6c154ab
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Aug 16 15:01:18 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Aug 16 15:01:18 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/ConfigRedactor.java | 84 ++++++++++++++++++++
.../apache/hadoop/conf/ReconfigurableBase.java | 13 ++-
.../fs/CommonConfigurationKeysPublic.java | 14 ++++
.../src/main/resources/core-default.xml | 10 +++
.../apache/hadoop/conf/TestConfigRedactor.java | 72 +++++++++++++++++
5 files changed, 190 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
new file mode 100644
index 0000000..0ba756c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
+
+/**
+ * Tool for redacting sensitive information when displaying config parameters.
+ *
+ * <p>Some config parameters contain sensitive information (for example, cloud
+ * storage keys). When these properties are displayed in plaintext, we should
+ * redactor their values as appropriate.
+ */
+public class ConfigRedactor {
+
+ private static final String REDACTED_TEXT = "<redacted>";
+
+ private List<Pattern> compiledPatterns;
+
+ public ConfigRedactor(Configuration conf) {
+ String sensitiveRegexList = conf.get(
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT);
+ List<String> sensitiveRegexes = Arrays.asList(sensitiveRegexList.split(","));
+ compiledPatterns = new ArrayList<Pattern>();
+ for (String regex : sensitiveRegexes) {
+ Pattern p = Pattern.compile(regex);
+ compiledPatterns.add(p);
+ }
+ }
+
+ /**
+ * Given a key / value pair, decides whether or not to redact and returns
+ * either the original value or text indicating it has been redacted.
+ *
+ * @param key
+ * @param value
+ * @return Original value, or text indicating it has been redacted
+ */
+ public String redact(String key, String value) {
+ if (configIsSensitive(key)) {
+ return REDACTED_TEXT;
+ }
+ return value;
+ }
+
+ /**
+ * Matches given config key against patterns and determines whether or not
+ * it should be considered sensitive enough to redact in logs and other
+ * plaintext displays.
+ *
+ * @param key
+ * @return True if parameter is considered sensitive
+ */
+ private boolean configIsSensitive(String key) {
+ for (Pattern regex : compiledPatterns) {
+ if (regex.matcher(key).find()) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
index 681ca2b..bdd006d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
@@ -117,17 +117,24 @@ public abstract class ReconfigurableBase
final Collection<PropertyChange> changes =
parent.getChangedProperties(newConf, oldConf);
Map<PropertyChange, Optional<String>> results = Maps.newHashMap();
+ ConfigRedactor oldRedactor = new ConfigRedactor(oldConf);
+ ConfigRedactor newRedactor = new ConfigRedactor(newConf);
for (PropertyChange change : changes) {
String errorMessage = null;
+ String oldValRedacted = oldRedactor.redact(change.prop, change.oldVal);
+ String newValRedacted = newRedactor.redact(change.prop, change.newVal);
if (!parent.isPropertyReconfigurable(change.prop)) {
LOG.info(String.format(
"Property %s is not configurable: old value: %s, new value: %s",
- change.prop, change.oldVal, change.newVal));
+ change.prop,
+ oldValRedacted,
+ newValRedacted));
continue;
}
LOG.info("Change property: " + change.prop + " from \""
- + ((change.oldVal == null) ? "<default>" : change.oldVal)
- + "\" to \"" + ((change.newVal == null) ? "<default>" : change.newVal)
+ + ((change.oldVal == null) ? "<default>" : oldValRedacted)
+ + "\" to \""
+ + ((change.newVal == null) ? "<default>" : newValRedacted)
+ "\".");
try {
String effectiveValue =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e746f2b..0a3afb7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -753,5 +753,19 @@ public class CommonConfigurationKeysPublic {
*/
public static final String HADOOP_SECURITY_CREDENTIAL_PASSWORD_FILE_KEY =
"hadoop.security.credstore.java-keystore-provider.password-file";
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
+ "hadoop.security.sensitive-config-keys";
+ public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
+ "password$" + "," +
+ "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
+ "fs.azure\\.account.key.*" + "," +
+ "dfs.webhdfs.oauth2.[a-z]+.token" + "," +
+ HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f9c3f72..e78795c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -503,6 +503,16 @@
</property>
<property>
+ <name>hadoop.security.sensitive-config-keys</name>
+ <value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
+ <description>A comma-separated list of regular expressions to match against
+ configuration keys that should be redacted where appropriate, for
+ example, when logging modified properties during a reconfiguration,
+ private credentials should not be logged.
+ </description>
+</property>
+
+<property>
<name>hadoop.workaround.non.threadsafe.getpwuid</name>
<value>true</value>
<description>Some operating systems or authentication modules are known to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b689e7a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
new file mode 100644
index 0000000..81f8f71
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.conf;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Tests the tool (and the default expression) for deciding which config
+ * redact.
+ */
+public class TestConfigRedactor {
+ private static final String REDACTED_TEXT = "<redacted>";
+
+ private static final String ORIGINAL_VALUE = "Hello, World!";
+
+ @Test
+ public void redact() throws Exception {
+ Configuration conf = new Configuration();
+ ConfigRedactor redactor = new ConfigRedactor(conf);
+ String processedText;
+
+ List<String> sensitiveKeys = Arrays.asList(
+ "fs.s3a.secret.key",
+ "fs.s3n.awsSecretKey",
+ "fs.azure.account.key.abcdefg.blob.core.windows.net",
+ "dfs.webhdfs.oauth2.access.token",
+ "dfs.webhdfs.oauth2.refresh.token",
+ "ssl.server.keystore.keypassword",
+ "ssl.server.keystore.password",
+ "hadoop.security.sensitive-config-keys"
+ );
+ for (String key : sensitiveKeys) {
+ processedText = redactor.redact(key, ORIGINAL_VALUE);
+ Assert.assertEquals(
+ "Config parameter wasn't redacted and should be: " + key,
+ REDACTED_TEXT, processedText);
+ }
+
+ List<String> normalKeys = Arrays.asList(
+ "fs.defaultFS",
+ "dfs.replication",
+ "ssl.server.keystore.location",
+ "hadoop.security.credstore.java-keystore-provider.password-file"
+ );
+ for (String key : normalKeys) {
+ processedText = redactor.redact(key, ORIGINAL_VALUE);
+ Assert.assertEquals(
+ "Config parameter was redacted and shouldn't be: " + key,
+ ORIGINAL_VALUE, processedText);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org