You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/10/07 00:06:05 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-14908. CrossOriginFilter
should trigger regex on more input (Johannes Alberti via aw) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 5c775e61e -> b8c68695a (forced update)
HADOOP-14908. CrossOriginFilter should trigger regex on more input (Johannes Alberti via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d5dd75b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d5dd75b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d5dd75b
Branch: refs/heads/YARN-5734
Commit: 4d5dd75b607d25adf8b41f7408713dfcea8f5330
Parents: 4111e6c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Oct 3 10:58:28 2017 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Oct 3 10:58:28 2017 -0700
----------------------------------------------------------------------
.../hadoop/security/http/CrossOriginFilter.java | 27 +++++--
.../src/main/resources/core-default.xml | 12 ++-
.../src/site/markdown/HttpAuthentication.md | 2 +-
.../security/http/TestCrossOriginFilter.java | 79 ++++++++++++++++++++
.../src/site/markdown/TimelineServer.md | 32 ++++----
5 files changed, 125 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
index 58d50cf..34d9fe2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/CrossOriginFilter.java
@@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import com.google.common.annotations.VisibleForTesting;
+import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,6 +67,7 @@ public class CrossOriginFilter implements Filter {
// Filter configuration
public static final String ALLOWED_ORIGINS = "allowed-origins";
public static final String ALLOWED_ORIGINS_DEFAULT = "*";
+ public static final String ALLOWED_ORIGINS_REGEX_PREFIX = "regex:";
public static final String ALLOWED_METHODS = "allowed-methods";
public static final String ALLOWED_METHODS_DEFAULT = "GET,POST,HEAD";
public static final String ALLOWED_HEADERS = "allowed-headers";
@@ -194,6 +196,12 @@ public class CrossOriginFilter implements Filter {
allowAllOrigins = allowedOrigins.contains("*");
LOG.info("Allowed Origins: " + StringUtils.join(allowedOrigins, ','));
LOG.info("Allow All Origins: " + allowAllOrigins);
+ List<String> discouragedAllowedOrigins = allowedOrigins.stream()
+ .filter(s -> s.length() > 1 && s.contains("*"))
+ .collect(Collectors.toList());
+ for (String discouragedAllowedOrigin : discouragedAllowedOrigins) {
+ LOG.warn("Allowed Origin pattern '" + discouragedAllowedOrigin + "' is discouraged, use the 'regex:' prefix and use a Java regular expression instead.");
+ }
}
private void initializeMaxAge(FilterConfig filterConfig) {
@@ -228,15 +236,20 @@ public class CrossOriginFilter implements Filter {
String[] origins = originsList.trim().split("\\s+");
for (String origin : origins) {
for (String allowedOrigin : allowedOrigins) {
- if (allowedOrigin.contains("*")) {
- String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
- Pattern p = Pattern.compile(regex);
- Matcher m = p.matcher(origin);
- if (m.matches()) {
+ Pattern regexPattern = null;
+ if (allowedOrigin.startsWith(ALLOWED_ORIGINS_REGEX_PREFIX)) {
+ String regex = allowedOrigin.substring(ALLOWED_ORIGINS_REGEX_PREFIX.length());
+ regexPattern = Pattern.compile(regex);
+ } else if (allowedOrigin.contains("*")) {
+ String regex = allowedOrigin.replace(".", "\\.").replace("*", ".*");
+ regexPattern = Pattern.compile(regex);
+ }
+
+ if (regexPattern != null
+ && regexPattern.matcher(origin).matches()) {
return true;
- }
} else if (allowedOrigin.equals(origin)) {
- return true;
+ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index a11e7c3..d481048 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1861,9 +1861,15 @@
<property>
<name>hadoop.http.cross-origin.allowed-origins</name>
<value>*</value>
- <description>Comma separated list of origins that are allowed for web
- services needing cross-origin (CORS) support. Wildcards (*) and patterns
- allowed</description>
+ <description>Comma separated list of origins that are allowed for web services
+ needing cross-origin (CORS) support. If a value in the list contains an
+ asterix (*), a regex pattern, escaping any dots ('.' -> '\.') and replacing
+ the asterix such that it captures any characters ('*' -> '.*'), is generated.
+ Values prefixed with 'regex:' are interpreted directly as regular expressions,
+ e.g. use the expression 'regex:https?:\/\/foo\.bar:([0-9]+)?' to allow any
+ origin using the 'http' or 'https' protocol in the domain 'foo.bar' on any
+ port. The use of simple wildcards ('*') is discouraged, and only available for
+ backward compatibility.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
index d4eeb0e..44d814c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
@@ -60,7 +60,7 @@ Add org.apache.hadoop.security.HttpCrossOriginFilterInitializer to hadoop.http.f
| Property | Default Value | Description |
|:---------------------------------------- |:--------------------------------------------- |:------------------------------------------------------------------------------------- |
| hadoop.http.cross-origin.enabled | `false` | Enables cross origin support for all web-services |
-| hadoop.http.cross-origin.allowed-origins | `*` | Comma separated list of origins that are allowed, wildcards (`*`) and patterns allowed |
+| hadoop.http.cross-origin.allowed-origins | `*` | Comma separated list of origins that are allowed. Values prefixed with `regex:` are interpreted as regular expressions. Values containing wildcards (`*`) are possible as well, here a regular expression is generated, the use is discouraged and support is only available for backward compatibility. |
| hadoop.http.cross-origin.allowed-methods | `GET,POST,HEAD` | Comma separated list of methods that are allowed |
| hadoop.http.cross-origin.allowed-headers | `X-Requested-With,Content-Type,Accept,Origin` | Comma separated list of headers that are allowed |
| hadoop.http.cross-origin.max-age | `1800` | Number of seconds a pre-flighted request can be cached |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
index 5c9b413..b9662b8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java
@@ -128,6 +128,85 @@ public class TestCrossOriginFilter {
}
@Test
+ public void testRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:.*[.]example[.]com");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ // match multiple sub-domains
+ Assert.assertFalse(filter.areOriginsAllowed("example.com"));
+ Assert.assertFalse(filter.areOriginsAllowed("foo:example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.bar.example.com"));
+
+ // First origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example.com foo.nomatch.com"));
+ // Second origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.nomatch.com foo.example.com"));
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
+ public void testComplexRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:https?:\\/\\/sub1[.]example[.]com(:[0-9]+)?");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com:1234"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com:8080"));
+
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
+ public void testMixedRegexPatternMatchingOrigins() throws ServletException, IOException {
+
+ // Setup the configuration settings of the server
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put(CrossOriginFilter.ALLOWED_ORIGINS, "regex:https?:\\/\\/sub1[.]example[.]com(:[0-9]+)?, "
+ + "*.example2.com");
+ FilterConfig filterConfig = new FilterConfigTest(conf);
+
+ // Object under test
+ CrossOriginFilter filter = new CrossOriginFilter();
+ filter.init(filterConfig);
+
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("http://sub1.example.com:1234"));
+ Assert.assertTrue(filter.areOriginsAllowed("https://sub1.example.com:8080"));
+
+ // match multiple sub-domains
+ Assert.assertFalse(filter.areOriginsAllowed("example2.com"));
+ Assert.assertFalse(filter.areOriginsAllowed("foo:example2.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example2.com"));
+ Assert.assertTrue(filter.areOriginsAllowed("foo.bar.example2.com"));
+
+ // First origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.example2.com foo.nomatch.com"));
+ // Second origin is allowed
+ Assert.assertTrue(filter.areOriginsAllowed("foo.nomatch.com foo.example2.com"));
+ // No origin in list is allowed
+ Assert.assertFalse(filter.areOriginsAllowed("foo.nomatch1.com foo.nomatch2.com"));
+ }
+
+ @Test
public void testDisallowedOrigin() throws ServletException, IOException {
// Setup the configuration settings of the server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d5dd75b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index f610cde..a326f45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -41,9 +41,9 @@ Previously this was supported purely for MapReduce jobs by the Application Histo
With the introduction of the timeline server, the Application History Server becomes just one use of
the Timeline Server.
-Generic information includes application level data such as
+Generic information includes application level data such as
-* queue-name,
+* queue-name,
* user information and the like set in the `ApplicationSubmissionContext`,
* a list of application-attempts that ran for an application
* information about each application-attempt
@@ -68,7 +68,7 @@ Current status
1. The "Timeline Server v1" REST API has been declared one of the REST APIs
whose compatibility will be maintained in future releases.
1. The single-server implementation of the Timeline Server places a limit on
- the scalability of the service; it also prevents the service being
+ the scalability of the service; it also prevents the service being
High-Availability component of the YARN infrastructure.
Future Plans
@@ -88,9 +88,9 @@ data structures as well as the ability of the client to failover between Timelin
The Timeline Domain offers a namespace for Timeline server allowing
users to host multiple entities, isolating them from other users and applications.
Timeline server Security is defined at this level.
-
+
A "Domain" primarily stores owner info, read and& write ACL information,
-created and modified time stamp information. Each Domain is identified by an ID which
+created and modified time stamp information. Each Domain is identified by an ID which
must be unique across all users in the YARN cluster.
#### Timeline Entity
@@ -111,7 +111,7 @@ Each Entity is uniquely identified by an `EntityId` and `EntityType`.
#### Timeline Events
A Timeline Event describes an event that is related to a specific
-Timeline Entity of an application.
+Timeline Entity of an application.
Users are free to define what an event means —such as starting
an application, getting allocated a container,
@@ -156,7 +156,7 @@ and cluster operators.
| `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
| `yarn.timeline-service.bind-host` | The actual address the server will bind to. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in `yarn.timeline-service.address` and `yarn.timeline-service.webapp.address`, respectively. This is most useful for making the service listen on all interfaces by setting to `0.0.0.0`. |
| `yarn.timeline-service.http-cross-origin.enabled` | Enables cross-origin support (CORS) for web services where cross-origin web response headers are needed. For example, javascript making a web services request to the timeline server. Defaults to `false`. |
-| `yarn.timeline-service.http-cross-origin.allowed-origins` | Comma separated list of origins that are allowed for web services needing cross-origin (CORS) support. Wildcards `(*)` and patterns allowed. Defaults to `*`. |
+| `yarn.timeline-service.http-cross-origin.allowed-origins` | Comma separated list of origins that are allowed. Values prefixed with `regex:` are interpreted as regular expressions. Values containing wildcards (`*`) are possible as well, here a regular expression is generated, the use is discouraged and support is only available for backward compatibility. Defaults to `*`. |
| `yarn.timeline-service.http-cross-origin.allowed-methods` | Comma separated list of methods that are allowed for web services needing cross-origin (CORS) support. Defaults to `GET,POST,HEAD`. |
| `yarn.timeline-service.http-cross-origin.allowed-headers` | Comma separated list of headers that are allowed for web services needing cross-origin (CORS) support. Defaults to `X-Requested-With,Content-Type,Accept,Origin`. |
| `yarn.timeline-service.http-cross-origin.max-age` | The number of seconds a pre-flighted request can be cached for web services needing cross-origin (CORS) support. Defaults to `1800`. |
@@ -420,7 +420,7 @@ response: `TimelinePutResponse`
### List domains of a user: GET `/ws/v1/timeline/domain`
-Retrieves a list of all domains of a user.
+Retrieves a list of all domains of a user.
If an owner is specified, that owner name overrides that of the caller.
@@ -598,8 +598,8 @@ Request Body:
Required fields
Entity: `type` and `id`. `starttime` is required unless the
-entity contains one or more event).
-Event: `type` and `timestamp`.
+entity contains one or more event).
+Event: `type` and `timestamp`.
## <a name="REST_API_LIST_TIMELINE_ENTITIES"></a>Timeline Entity List
@@ -809,7 +809,7 @@ Response Body:
}
]
}
-
+
@@ -1443,8 +1443,8 @@ None
### Elements of the `appattempts` (Application Attempt List) Object
When you make a request for the list of application attempts, the information
-will be returned as a collection of application attempt objects. See
-[Application Attempt](#REST_API_APPLICATION_ATTEMPT) for the syntax of
+will be returned as a collection of application attempt objects. See
+[Application Attempt](#REST_API_APPLICATION_ATTEMPT) for the syntax of
the application attempt object.
| Item | Data Type | Description |
@@ -1758,7 +1758,7 @@ Response Body:
}
]
}
-
+
#### XML response
@@ -2004,8 +2004,8 @@ Response Body:
<containerState>COMPLETE</containerState>
<nodeHttpAddress>http://localhost:8042</nodeHttpAddress>
</container>
-
-
+
+
### Response Codes
1. Queries where a domain, entity type, entity ID or similar cannot be resolved result in
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-6322: Disable queue refresh
when configuration mutation is enabled. Contributed by Jonathan Hung
Posted by jh...@apache.org.
YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc53a0c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc53a0c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc53a0c8
Branch: refs/heads/YARN-5734
Commit: dc53a0c8f0ddf7ca4f2f6564eee113550d81d418
Parents: f127c54
Author: Xuan <xg...@apache.org>
Authored: Tue Aug 1 08:48:04 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../server/resourcemanager/AdminService.java | 12 +++++++++-
.../scheduler/MutableConfScheduler.java | 12 ++++++++--
.../scheduler/MutableConfigurationProvider.java | 4 +++-
.../scheduler/capacity/CapacityScheduler.java | 9 ++++++--
.../conf/MutableCSConfigurationProvider.java | 11 +++++-----
.../resourcemanager/TestRMAdminService.java | 23 ++++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 14 +++++++-----
7 files changed, 67 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 3457ae3..fd9e849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol;
@@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst
import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import com.google.common.annotations.VisibleForTesting;
@@ -384,6 +387,12 @@ public class AdminService extends CompositeService implements
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ if (scheduler instanceof MutableConfScheduler
+ && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+ throw new IOException("Scheduler configuration is mutable. " +
+ operation + " is not allowed in this scenario.");
+ }
refreshQueues();
RMAuditLogger.logSuccess(user.getShortUserName(), operation,
"AdminService");
@@ -393,7 +402,8 @@ public class AdminService extends CompositeService implements
}
}
- private void refreshQueues() throws IOException, YarnException {
+ @Private
+ public void refreshQueues() throws IOException, YarnException {
rm.getRMContext().getScheduler().reinitialize(getConfig(),
this.rm.getRMContext());
// refresh the reservation system
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 007dc29..313bf6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -33,10 +34,11 @@ public interface MutableConfScheduler extends ResourceScheduler {
* Update the scheduler's configuration.
* @param user Caller of this update
* @param confUpdate configuration update
- * @throws IOException if update is invalid
+ * @throws IOException if scheduler could not be reinitialized
+ * @throws YarnException if reservation system could not be reinitialized
*/
void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException;
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException;
/**
* Get the scheduler configuration.
@@ -50,4 +52,10 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @return the queue object
*/
Queue getQueue(String queueName);
+
+ /**
+ * Return whether the scheduler configuration is mutable.
+ * @return whether scheduler configuration is mutable or not.
+ */
+ boolean isConfigurationMutable();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 1f13467..9baf1ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -39,8 +40,9 @@ public interface MutableConfigurationProvider {
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
+ * @throws YarnException if reservation system could not be reinitialized
*/
void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
- confUpdate) throws IOException;
+ confUpdate) throws IOException, YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 6d2de7e..51ee6a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2620,8 +2620,8 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException {
- if (csConfProvider instanceof MutableConfigurationProvider) {
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
+ if (isConfigurationMutable()) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
user, confUpdate);
} else {
@@ -2629,4 +2629,9 @@ public class CapacityScheduler extends
"provider does not support updating configuration.");
}
}
+
+ @Override
+ public boolean isConfigurationMutable() {
+ return csConfProvider instanceof MutableConfigurationProvider;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 9ccc146..d03b2e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
@@ -58,7 +59,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
- private Configuration conf;
public MutableCSConfigurationProvider(RMContext rmContext) {
this.rmContext = rmContext;
@@ -96,7 +96,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
.getPolicy(config);
aclMutationPolicy.init(config, rmContext);
- this.conf = config;
}
@Override
@@ -109,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public synchronized void mutateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
" queues.");
@@ -126,8 +125,8 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
}
try {
- rmContext.getScheduler().reinitialize(conf, rmContext);
- } catch (IOException e) {
+ rmContext.getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
throw e;
@@ -148,7 +147,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
}
try {
- rmContext.getScheduler().reinitialize(conf, rmContext);
+ rmContext.getScheduler().reinitialize(schedConf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(mutation.getId(), false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 588f16d..620ca58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -198,6 +198,29 @@ public class TestRMAdminService {
}
@Test
+ public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() {
+ configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+
+ try {
+ rm = new MockRM(configuration);
+ rm.init(configuration);
+ rm.start();
+ } catch (Exception ex) {
+ fail("Should not get any exceptions");
+ }
+
+ try {
+ rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
+ fail("Expected exception while calling refreshQueues when scheduler" +
+ " configuration is mutable.");
+ } catch (Exception ex) {
+ assertTrue(ex.getMessage().endsWith("Scheduler configuration is " +
+ "mutable. refreshQueues is not allowed in this scenario."));
+ }
+ }
+
+ @Test
public void testAdminRefreshNodesWithoutConfiguration()
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc53a0c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 9104f16..635a184 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -34,7 +36,6 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -50,6 +51,7 @@ public class TestMutableCSConfigurationProvider {
private SchedConfUpdateInfo goodUpdate;
private SchedConfUpdateInfo badUpdate;
private CapacityScheduler cs;
+ private AdminService adminService;
private static final UserGroupInformation TEST_USER = UserGroupInformation
.createUserForTesting("testUser", new String[] {});
@@ -61,6 +63,8 @@ public class TestMutableCSConfigurationProvider {
when(rmContext.getScheduler()).thenReturn(cs);
when(cs.getConfiguration()).thenReturn(
new CapacitySchedulerConfiguration());
+ adminService = mock(AdminService.class);
+ when(rmContext.getRMAdminService()).thenReturn(adminService);
confProvider = new MutableCSConfigurationProvider(rmContext);
goodUpdate = new SchedConfUpdateInfo();
Map<String, String> goodUpdateMap = new HashMap<>();
@@ -78,22 +82,20 @@ public class TestMutableCSConfigurationProvider {
}
@Test
- public void testInMemoryBackedProvider() throws IOException {
+ public void testInMemoryBackedProvider() throws IOException, YarnException {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
- doNothing().when(cs).reinitialize(any(Configuration.class),
- any(RMContext.class));
+ doNothing().when(adminService).refreshQueues();
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
- doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
- any(RMContext.class));
+ doThrow(new IOException()).when(adminService).refreshQueues();
try {
confProvider.mutateConfiguration(TEST_USER, badUpdate);
} catch (IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-7258. Add Node and Rack Hints
to Opportunistic Scheduler. (Kartheek Muthyala via asuresh).
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocationE2E.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocationE2E.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocationE2E.java
new file mode 100644
index 0000000..94cb28e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocationE2E.java
@@ -0,0 +1,784 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.NMTokenCache;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Class that tests the allocation of OPPORTUNISTIC containers through the
+ * centralized ResourceManager.
+ */
+public class TestOpportunisticContainerAllocationE2E {
+ private static Configuration conf = null;
+ private static MiniYARNCluster yarnCluster = null;
+ private static YarnClient yarnClient = null;
+ private static List<NodeReport> nodeReports = null;
+ private static int nodeCount = 3;
+
+ private static final int ROLLING_INTERVAL_SEC = 13;
+ private static final long AM_EXPIRE_MS = 4000;
+
+ private static Resource capability;
+ private static ProfileCapability profileCapability;
+ private static Priority priority;
+ private static Priority priority2;
+ private static Priority priority3;
+ private static Priority priority4;
+ private static String node;
+ private static String rack;
+ private static String[] nodes;
+ private static String[] racks;
+ private final static int DEFAULT_ITERATION = 3;
+
+ // Per test..
+ private ApplicationAttemptId attemptId = null;
+ private AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
+ private long availMB;
+ private int availVCores;
+ private long allocMB;
+ private int allocVCores;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ // start minicluster
+ conf = new YarnConfiguration();
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ ROLLING_INTERVAL_SEC);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS);
+ conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1000);
+ // set the minimum allocation so that resource decrease can go under 1024
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setBoolean(
+ YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH, 10);
+ conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
+ yarnCluster =
+ new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ // start rm client
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // get node info
+ nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
+
+ priority = Priority.newInstance(1);
+ priority2 = Priority.newInstance(2);
+ priority3 = Priority.newInstance(3);
+ priority4 = Priority.newInstance(4);
+ capability = Resource.newInstance(512, 1);
+ profileCapability = ProfileCapability.newInstance(capability);
+
+ node = nodeReports.get(0).getNodeId().getHost();
+ rack = nodeReports.get(0).getRackName();
+ nodes = new String[]{node};
+ racks = new String[]{rack};
+ }
+
+ @Before
+ public void startApp() throws Exception {
+ // submit new app
+ ApplicationSubmissionContext appContext =
+ yarnClient.createApplication().getApplicationSubmissionContext();
+ ApplicationId appId = appContext.getApplicationId();
+ // set the application name
+ appContext.setApplicationName("Test");
+ // Set the priority for the application master
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(0);
+ appContext.setPriority(pri);
+ // Set the queue to which this application is to be submitted in the RM
+ appContext.setQueue("default");
+ // Set up the container launch context for the application master
+ ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext(
+ Collections.<String, LocalResource>emptyMap(),
+ new HashMap<String, String>(), Arrays.asList("sleep", "100"),
+ new HashMap<String, ByteBuffer>(), null,
+ new HashMap<ApplicationAccessType, String>());
+ appContext.setAMContainerSpec(amContainer);
+ appContext.setResource(Resource.newInstance(1024, 1));
+ // Create the request to send to the applications manager
+ SubmitApplicationRequest appRequest =
+ Records.newRecord(SubmitApplicationRequest.class);
+ appRequest.setApplicationSubmissionContext(appContext);
+ // Submit the application to the applications manager
+ yarnClient.submitApplication(appContext);
+
+ // wait for app to start
+ RMAppAttempt appAttempt = null;
+ while (true) {
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+ if (appReport.getYarnApplicationState() ==
+ YarnApplicationState.ACCEPTED) {
+ attemptId = appReport.getCurrentApplicationAttemptId();
+ appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
+ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
+ while (true) {
+ if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
+ break;
+ }
+ }
+ break;
+ }
+ }
+ // Just dig into the ResourceManager and get the AMRMToken just for the sake
+ // of testing.
+ UserGroupInformation.setLoginUser(UserGroupInformation
+ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
+
+ // emulate RM setup of AMRM token in credentials by adding the token
+ // *before* setting the token service
+ UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+ appAttempt.getAMRMToken()
+ .setService(ClientRMProxy.getAMRMTokenService(conf));
+
+ // start am rm client
+ amClient = (AMRMClientImpl<AMRMClient.ContainerRequest>)AMRMClient
+ .createAMRMClient();
+
+ //setting an instance NMTokenCache
+ amClient.setNMTokenCache(new NMTokenCache());
+ //asserting we are not using the singleton instance cache
+ Assert.assertNotSame(NMTokenCache.getSingleton(),
+ amClient.getNMTokenCache());
+
+ amClient.init(conf);
+ amClient.start();
+
+ amClient.registerApplicationMaster("Host", 10000, "");
+ }
+
+ @After
+ public void cancelApp() throws YarnException, IOException {
+ try {
+ amClient
+ .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null,
+ null);
+ } finally {
+ if (amClient != null &&
+ amClient.getServiceState() == Service.STATE.STARTED) {
+ amClient.stop();
+ }
+ }
+ yarnClient.killApplication(attemptId.getApplicationId());
+ attemptId = null;
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (yarnClient != null &&
+ yarnClient.getServiceState() == Service.STATE.STARTED) {
+ yarnClient.stop();
+ }
+ if (yarnCluster != null &&
+ yarnCluster.getServiceState() == Service.STATE.STARTED) {
+ yarnCluster.stop();
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testPromotionFromAcquired() throws YarnException, IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+
+ int oppContainersRequestedAny =
+ amClient.getTable(0).get(priority2, ResourceRequest.ANY,
+ ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
+ .getNumContainers();
+
+ assertEquals(1, oppContainersRequestedAny);
+
+ assertEquals(1, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ Map<ContainerId, Container> allocatedOpportContainers = new HashMap<>();
+ int iterationsLeft = 50;
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ updateMetrics("Before Opp Allocation");
+
+ while (allocatedContainerCount < oppContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
+ allocatedOpportContainers.put(container.getId(), container);
+ removeCR(container);
+ }
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < oppContainersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(oppContainersRequestedAny, allocatedContainerCount);
+ assertEquals(oppContainersRequestedAny, allocatedOpportContainers.size());
+
+ updateMetrics("After Opp Allocation / Before Promotion");
+
+ try {
+ Container c = allocatedOpportContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.OPPORTUNISTIC));
+ Assert.fail("Should throw Exception..");
+ } catch (IllegalArgumentException e) {
+ System.out.println("## " + e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "target should be GUARANTEED and original should be OPPORTUNISTIC"));
+ }
+
+ Container c = allocatedOpportContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED));
+ iterationsLeft = 120;
+ Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
+ // do a few iterations to ensure RM is not going to send new containers
+ while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
+ // inform RM of rejection
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ // RM did not send new containers because AM does not need any
+ if (allocResponse.getUpdatedContainers() != null) {
+ for (UpdatedContainer updatedContainer : allocResponse
+ .getUpdatedContainers()) {
+ System.out.println("Got update..");
+ updatedContainers.put(updatedContainer.getContainer().getId(),
+ updatedContainer);
+ }
+ }
+ if (iterationsLeft > 0) {
+ // sleep to make sure NM's heartbeat
+ sleep(100);
+ }
+ }
+
+ updateMetrics("After Promotion");
+
+ assertEquals(1, updatedContainers.size());
+ for (ContainerId cId : allocatedOpportContainers.keySet()) {
+ Container orig = allocatedOpportContainers.get(cId);
+ UpdatedContainer updatedContainer = updatedContainers.get(cId);
+ assertNotNull(updatedContainer);
+ assertEquals(ExecutionType.GUARANTEED,
+ updatedContainer.getContainer().getExecutionType());
+ assertEquals(orig.getResource(),
+ updatedContainer.getContainer().getResource());
+ assertEquals(orig.getNodeId(),
+ updatedContainer.getContainer().getNodeId());
+ assertEquals(orig.getVersion() + 1,
+ updatedContainer.getContainer().getVersion());
+ }
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+ amClient.ask.clear();
+ }
+
+ @Test(timeout = 60000)
+ public void testDemotionFromAcquired() throws YarnException, IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority3));
+
+ int guarContainersRequestedAny = amClient.getTable(0).get(priority3,
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
+ .remoteRequest.getNumContainers();
+
+ assertEquals(1, guarContainersRequestedAny);
+
+ assertEquals(1, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ Map<ContainerId, Container> allocatedGuarContainers = new HashMap<>();
+ int iterationsLeft = 50;
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ updateMetrics("Before Guar Allocation");
+
+ while (allocatedContainerCount < guarContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ allocatedGuarContainers.put(container.getId(), container);
+ removeCR(container);
+ }
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < guarContainersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(guarContainersRequestedAny, allocatedContainerCount);
+ assertEquals(guarContainersRequestedAny, allocatedGuarContainers.size());
+
+ updateMetrics("After Guar Allocation / Before Demotion");
+
+ try {
+ Container c = allocatedGuarContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ null, ExecutionType.GUARANTEED));
+ Assert.fail("Should throw Exception..");
+ } catch (IllegalArgumentException e) {
+ System.out.println("## " + e.getMessage());
+ Assert.assertTrue(e.getMessage().contains(
+ "target should be OPPORTUNISTIC and original should be GUARANTEED"));
+ }
+
+ Container c = allocatedGuarContainers.values().iterator().next();
+ amClient.requestContainerUpdate(
+ c, UpdateContainerRequest.newInstance(c.getVersion(),
+ c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
+ null, ExecutionType.OPPORTUNISTIC));
+ iterationsLeft = 120;
+ Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
+ // do a few iterations to ensure RM is not going to send new containers
+ while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
+ // inform RM of rejection
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ // RM did not send new containers because AM does not need any
+ if (allocResponse.getUpdatedContainers() != null) {
+ for (UpdatedContainer updatedContainer : allocResponse
+ .getUpdatedContainers()) {
+ System.out.println("Got update..");
+ updatedContainers.put(updatedContainer.getContainer().getId(),
+ updatedContainer);
+ }
+ }
+ if (iterationsLeft > 0) {
+ // sleep to make sure NM's heartbeat
+ sleep(100);
+ }
+ }
+
+ updateMetrics("After Demotion");
+
+ assertEquals(1, updatedContainers.size());
+ for (ContainerId cId : allocatedGuarContainers.keySet()) {
+ Container orig = allocatedGuarContainers.get(cId);
+ UpdatedContainer updatedContainer = updatedContainers.get(cId);
+ assertNotNull(updatedContainer);
+ assertEquals(ExecutionType.OPPORTUNISTIC,
+ updatedContainer.getContainer().getExecutionType());
+ assertEquals(orig.getResource(),
+ updatedContainer.getContainer().getResource());
+ assertEquals(orig.getNodeId(),
+ updatedContainer.getContainer().getNodeId());
+ assertEquals(orig.getVersion() + 1,
+ updatedContainer.getContainer().getVersion());
+ }
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+ amClient.ask.clear();
+ }
+
+ @Test(timeout = 60000)
+ public void testMixedAllocationAndRelease() throws YarnException,
+ IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+
+ int containersRequestedNode = amClient.getTable(0).get(priority,
+ node, ExecutionType.GUARANTEED, profileCapability).remoteRequest
+ .getNumContainers();
+ int containersRequestedRack = amClient.getTable(0).get(priority,
+ rack, ExecutionType.GUARANTEED, profileCapability).remoteRequest
+ .getNumContainers();
+ int containersRequestedAny = amClient.getTable(0).get(priority,
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
+ .remoteRequest.getNumContainers();
+ int oppContainersRequestedAny =
+ amClient.getTable(0).get(priority2, ResourceRequest.ANY,
+ ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
+ .getNumContainers();
+
+ assertEquals(4, containersRequestedNode);
+ assertEquals(4, containersRequestedRack);
+ assertEquals(4, containersRequestedAny);
+ assertEquals(2, oppContainersRequestedAny);
+
+ assertEquals(4, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+
+ containersRequestedNode = amClient.getTable(0).get(priority,
+ node, ExecutionType.GUARANTEED, profileCapability).remoteRequest
+ .getNumContainers();
+ containersRequestedRack = amClient.getTable(0).get(priority,
+ rack, ExecutionType.GUARANTEED, profileCapability).remoteRequest
+ .getNumContainers();
+ containersRequestedAny = amClient.getTable(0).get(priority,
+ ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
+ .remoteRequest.getNumContainers();
+ oppContainersRequestedAny =
+ amClient.getTable(0).get(priority2, ResourceRequest.ANY,
+ ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
+ .getNumContainers();
+
+ assertEquals(2, containersRequestedNode);
+ assertEquals(2, containersRequestedRack);
+ assertEquals(2, containersRequestedAny);
+ assertEquals(1, oppContainersRequestedAny);
+
+ assertEquals(4, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ int allocatedOpportContainerCount = 0;
+ int iterationsLeft = 50;
+ Set<ContainerId> releases = new TreeSet<>();
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ while (allocatedContainerCount <
+ containersRequestedAny + oppContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ allocatedContainerCount +=
+ allocResponse.getAllocatedContainers().size();
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
+ allocatedOpportContainerCount++;
+ }
+ ContainerId rejectContainerId = container.getId();
+ releases.add(rejectContainerId);
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < containersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(containersRequestedAny + oppContainersRequestedAny,
+ allocatedContainerCount);
+ assertEquals(oppContainersRequestedAny, allocatedOpportContainerCount);
+ for (ContainerId rejectContainerId : releases) {
+ amClient.releaseAssignedContainer(rejectContainerId);
+ }
+ assertEquals(3, amClient.release.size());
+ assertEquals(0, amClient.ask.size());
+
+ // need to tell the AMRMClient that we don't need these resources anymore
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ assertEquals(4, amClient.ask.size());
+
+ iterationsLeft = 3;
+ // do a few iterations to ensure RM is not going to send new containers
+ while (iterationsLeft-- > 0) {
+ // inform RM of rejection
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ // RM did not send new containers because AM does not need any
+ assertEquals(0, allocResponse.getAllocatedContainers().size());
+ if (allocResponse.getCompletedContainersStatuses().size() > 0) {
+ for (ContainerStatus cStatus : allocResponse
+ .getCompletedContainersStatuses()) {
+ if (releases.contains(cStatus.getContainerId())) {
+ assertEquals(cStatus.getState(), ContainerState.COMPLETE);
+ assertEquals(-100, cStatus.getExitStatus());
+ releases.remove(cStatus.getContainerId());
+ }
+ }
+ }
+ if (iterationsLeft > 0) {
+ // sleep to make sure NM's heartbeat
+ sleep(100);
+ }
+ }
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+ }
+
+ /**
+ * Tests allocation with requests comprising only opportunistic containers.
+ */
+ @Test(timeout = 60000)
+ public void testOpportunisticAllocation() throws YarnException, IOException {
+ // setup container request
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ amClient.addContainerRequest(
+ new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
+ true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+
+ int oppContainersRequestedAny = amClient.getTable(0)
+ .get(priority3, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC,
+ profileCapability).remoteRequest.getNumContainers();
+
+ assertEquals(2, oppContainersRequestedAny);
+
+ assertEquals(1, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ // RM should allocate container within 2 calls to allocate()
+ int allocatedContainerCount = 0;
+ int iterationsLeft = 10;
+ Set<ContainerId> releases = new TreeSet<>();
+
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0,
+ amClient.getNMTokenCache().numberOfTokensInCache());
+ HashMap<String, Token> receivedNMTokens = new HashMap<>();
+
+ while (allocatedContainerCount < oppContainersRequestedAny
+ && iterationsLeft-- > 0) {
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertEquals(0, amClient.ask.size());
+ assertEquals(0, amClient.release.size());
+
+ for (Container container : allocResponse.getAllocatedContainers()) {
+ allocatedContainerCount++;
+ ContainerId rejectContainerId = container.getId();
+ releases.add(rejectContainerId);
+ }
+
+ for (NMToken token : allocResponse.getNMTokens()) {
+ String nodeID = token.getNodeId().toString();
+ receivedNMTokens.put(nodeID, token.getToken());
+ }
+
+ if (allocatedContainerCount < oppContainersRequestedAny) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+
+ assertEquals(oppContainersRequestedAny, allocatedContainerCount);
+ assertEquals(1, receivedNMTokens.values().size());
+ }
+
+ private void removeCR(Container container) {
+ List<? extends Collection<AMRMClient.ContainerRequest>>
+ matchingRequests = amClient.getMatchingRequests(container
+ .getPriority(),
+ ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC,
+ container.getResource());
+ Set<AMRMClient.ContainerRequest> toRemove = new HashSet<>();
+ for (Collection<AMRMClient.ContainerRequest> rc : matchingRequests) {
+ for (AMRMClient.ContainerRequest cr : rc) {
+ toRemove.add(cr);
+ }
+ }
+ for (AMRMClient.ContainerRequest cr : toRemove) {
+ amClient.removeContainerRequest(cr);
+ }
+ }
+
+ private void updateMetrics(String msg) {
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler)yarnCluster.getResourceManager()
+ .getResourceScheduler();
+ availMB = scheduler.getRootQueueMetrics().getAvailableMB();
+ availVCores = scheduler.getRootQueueMetrics().getAvailableVirtualCores();
+ allocMB = scheduler.getRootQueueMetrics().getAllocatedMB();
+ allocVCores = scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
+ System.out.println("## METRICS (" + msg + ")==>");
+ System.out.println(" : availMB=" + availMB + ", " +
+ "availVCores=" +availVCores + ", " +
+ "allocMB=" + allocMB + ", " +
+ "allocVCores=" + allocVCores + ", ");
+ System.out.println("<== ##");
+ }
+
+ private void sleep(int sleepTime) {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RemoteNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RemoteNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RemoteNode.java
index e403a12..f621aa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RemoteNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RemoteNode.java
@@ -47,6 +47,24 @@ public abstract class RemoteNode implements Comparable<RemoteNode> {
}
/**
+ * Create new Instance.
+ * @param nodeId NodeId.
+ * @param httpAddress Http address.
+ * @param rackName Rack Name.
+ * @return RemoteNode instance.
+ */
+ @Private
+ @Unstable
+ public static RemoteNode newInstance(NodeId nodeId, String httpAddress,
+ String rackName) {
+ RemoteNode remoteNode = Records.newRecord(RemoteNode.class);
+ remoteNode.setNodeId(nodeId);
+ remoteNode.setHttpAddress(httpAddress);
+ remoteNode.setRackName(rackName);
+ return remoteNode;
+ }
+
+ /**
* Get {@link NodeId}.
* @return NodeId.
*/
@@ -79,6 +97,22 @@ public abstract class RemoteNode implements Comparable<RemoteNode> {
public abstract void setHttpAddress(String httpAddress);
/**
+ * Get Rack Name.
+ * @return Rack Name.
+ */
+ @Private
+ @Unstable
+ public abstract String getRackName();
+
+ /**
+ * Set Rack Name.
+ * @param rackName Rack Name.
+ */
+ @Private
+ @Unstable
+ public abstract void setRackName(String rackName);
+
+ /**
* Use the underlying {@link NodeId} comparator.
* @param other RemoteNode.
* @return Comparison.
@@ -92,6 +126,7 @@ public abstract class RemoteNode implements Comparable<RemoteNode> {
public String toString() {
return "RemoteNode{" +
"nodeId=" + getNodeId() + ", " +
+ "rackName=" + getRackName() + ", " +
"httpAddress=" + getHttpAddress() + "}";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoteNodePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoteNodePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoteNodePBImpl.java
index 3e4fd4a..c2492cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoteNodePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoteNodePBImpl.java
@@ -118,6 +118,25 @@ public class RemoteNodePBImpl extends RemoteNode {
}
@Override
+ public String getRackName() {
+ RemoteNodeProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasRackName()) {
+ return null;
+ }
+ return (p.getRackName());
+ }
+
+ @Override
+ public void setRackName(String rackName) {
+ maybeInitBuilder();
+ if (rackName == null) {
+ builder.clearRackName();
+ return;
+ }
+ builder.setRackName(rackName);
+ }
+
+ @Override
public int hashCode() {
return getProto().hashCode();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index 782dc02..ede4958 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -45,11 +45,14 @@ import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -61,6 +64,10 @@ import java.util.concurrent.atomic.AtomicLong;
*/
public class OpportunisticContainerAllocator {
+ private static final int NODE_LOCAL_LOOP = 0;
+ private static final int RACK_LOCAL_LOOP = 1;
+ private static final int OFF_SWITCH_LOOP = 2;
+
/**
* This class encapsulates application specific parameters used to build a
* Container.
@@ -70,6 +77,7 @@ public class OpportunisticContainerAllocator {
private Resource minResource;
private Resource incrementResource;
private int containerTokenExpiryInterval;
+ private int maxAllocationsPerSchedulerKeyPerRound = 1;
/**
* Return Max Resource.
@@ -135,6 +143,24 @@ public class OpportunisticContainerAllocator {
int containerTokenExpiryInterval) {
this.containerTokenExpiryInterval = containerTokenExpiryInterval;
}
+
+ /**
+ * Get the Max Allocations per Scheduler Key per allocation round.
+ * @return maxAllocationsPerSchedulerKeyPerRound.
+ */
+ public int getMaxAllocationsPerSchedulerKeyPerRound() {
+ return maxAllocationsPerSchedulerKeyPerRound;
+ }
+
+ /**
+ * Set the Max Allocations per Scheduler Key per allocation round.
+ * @param maxAllocationsPerSchedulerKeyPerRound val.
+ */
+ public void setMaxAllocationsPerSchedulerKeyPerRound(
+ int maxAllocationsPerSchedulerKeyPerRound) {
+ this.maxAllocationsPerSchedulerKeyPerRound =
+ maxAllocationsPerSchedulerKeyPerRound;
+ }
}
/**
@@ -188,6 +214,72 @@ public class OpportunisticContainerAllocator {
private final BaseContainerTokenSecretManager tokenSecretManager;
+ static class Allocation {
+ private final Container container;
+ private final String resourceName;
+
+ Allocation(Container container, String resourceName) {
+ this.container = container;
+ this.resourceName = resourceName;
+ }
+
+ Container getContainer() {
+ return container;
+ }
+
+ String getResourceName() {
+ return resourceName;
+ }
+ }
+
+ static class EnrichedResourceRequest {
+ private final Map<String, AtomicInteger> nodeLocations = new HashMap<>();
+ private final Map<String, AtomicInteger> rackLocations = new HashMap<>();
+ private final ResourceRequest request;
+
+ EnrichedResourceRequest(ResourceRequest request) {
+ this.request = request;
+ }
+
+ ResourceRequest getRequest() {
+ return request;
+ }
+
+ void addLocation(String location, int count) {
+ Map<String, AtomicInteger> m = rackLocations;
+ if (!location.startsWith("/")) {
+ m = nodeLocations;
+ }
+ if (count == 0) {
+ m.remove(location);
+ } else {
+ m.put(location, new AtomicInteger(count));
+ }
+ }
+
+ void removeLocation(String location) {
+ Map<String, AtomicInteger> m = rackLocations;
+ AtomicInteger count = m.get(location);
+ if (count == null) {
+ m = nodeLocations;
+ count = m.get(location);
+ }
+
+ if (count != null) {
+ if (count.decrementAndGet() == 0) {
+ m.remove(location);
+ }
+ }
+ }
+
+ Set<String> getNodeLocations() {
+ return nodeLocations.keySet();
+ }
+
+ Set<String> getRackLocations() {
+ return rackLocations.keySet();
+ }
+ }
/**
* Create a new Opportunistic Container Allocator.
* @param tokenSecretManager TokenSecretManager
@@ -223,37 +315,55 @@ public class OpportunisticContainerAllocator {
// Add OPPORTUNISTIC requests to the outstanding ones.
opportContext.addToOutstandingReqs(oppResourceReqs);
- // Satisfy the outstanding OPPORTUNISTIC requests.
+ Set<String> nodeBlackList = new HashSet<>(opportContext.getBlacklist());
List<Container> allocatedContainers = new ArrayList<>();
- for (SchedulerRequestKey schedulerKey :
- opportContext.getOutstandingOpReqs().descendingKeySet()) {
- // Allocated containers :
- // Key = Requested Capability,
- // Value = List of Containers of given cap (the actual container size
- // might be different than what is requested, which is why
- // we need the requested capability (key) to match against
- // the outstanding reqs)
- Map<Resource, List<Container>> allocated = allocate(rmIdentifier,
- opportContext, schedulerKey, applicationAttemptId, appSubmitter);
- for (Map.Entry<Resource, List<Container>> e : allocated.entrySet()) {
- opportContext.matchAllocationToOutstandingRequest(
- e.getKey(), e.getValue());
- allocatedContainers.addAll(e.getValue());
+
+ // Satisfy the outstanding OPPORTUNISTIC requests.
+ boolean continueLoop = true;
+ while (continueLoop) {
+ continueLoop = false;
+ List<Map<Resource, List<Allocation>>> allocations = new ArrayList<>();
+ for (SchedulerRequestKey schedulerKey :
+ opportContext.getOutstandingOpReqs().descendingKeySet()) {
+ // Allocated containers :
+ // Key = Requested Capability,
+ // Value = List of Containers of given cap (the actual container size
+ // might be different than what is requested, which is why
+ // we need the requested capability (key) to match against
+ // the outstanding reqs)
+ Map<Resource, List<Allocation>> allocation = allocate(
+ rmIdentifier, opportContext, schedulerKey, applicationAttemptId,
+ appSubmitter, nodeBlackList);
+ if (allocation.size() > 0) {
+ allocations.add(allocation);
+ continueLoop = true;
+ }
+ }
+ for (Map<Resource, List<Allocation>> allocation : allocations) {
+ for (Map.Entry<Resource, List<Allocation>> e : allocation.entrySet()) {
+ opportContext.matchAllocationToOutstandingRequest(
+ e.getKey(), e.getValue());
+ for (Allocation alloc : e.getValue()) {
+ allocatedContainers.add(alloc.getContainer());
+ }
+ }
}
}
return allocatedContainers;
}
- private Map<Resource, List<Container>> allocate(long rmIdentifier,
+ private Map<Resource, List<Allocation>> allocate(long rmIdentifier,
OpportunisticContainerContext appContext, SchedulerRequestKey schedKey,
- ApplicationAttemptId appAttId, String userName) throws YarnException {
- Map<Resource, List<Container>> containers = new HashMap<>();
- for (ResourceRequest anyAsk :
+ ApplicationAttemptId appAttId, String userName, Set<String> blackList)
+ throws YarnException {
+ Map<Resource, List<Allocation>> containers = new HashMap<>();
+ for (EnrichedResourceRequest enrichedAsk :
appContext.getOutstandingOpReqs().get(schedKey).values()) {
allocateContainersInternal(rmIdentifier, appContext.getAppParams(),
- appContext.getContainerIdGenerator(), appContext.getBlacklist(),
- appAttId, appContext.getNodeMap(), userName, containers, anyAsk);
+ appContext.getContainerIdGenerator(), blackList, appAttId,
+ appContext.getNodeMap(), userName, containers, enrichedAsk);
+ ResourceRequest anyAsk = enrichedAsk.getRequest();
if (!containers.isEmpty()) {
LOG.info("Opportunistic allocation requested for [priority={}, "
+ "allocationRequestId={}, num_containers={}, capability={}] "
@@ -269,43 +379,162 @@ public class OpportunisticContainerAllocator {
AllocationParams appParams, ContainerIdGenerator idCounter,
Set<String> blacklist, ApplicationAttemptId id,
Map<String, RemoteNode> allNodes, String userName,
- Map<Resource, List<Container>> containers, ResourceRequest anyAsk)
+ Map<Resource, List<Allocation>> allocations,
+ EnrichedResourceRequest enrichedAsk)
throws YarnException {
+ if (allNodes.size() == 0) {
+ LOG.info("No nodes currently available to " +
+ "allocate OPPORTUNISTIC containers.");
+ return;
+ }
+ ResourceRequest anyAsk = enrichedAsk.getRequest();
int toAllocate = anyAsk.getNumContainers()
- - (containers.isEmpty() ? 0 :
- containers.get(anyAsk.getCapability()).size());
-
- List<RemoteNode> nodesForScheduling = new ArrayList<>();
- for (Entry<String, RemoteNode> nodeEntry : allNodes.entrySet()) {
- // Do not use blacklisted nodes for scheduling.
- if (blacklist.contains(nodeEntry.getKey())) {
- continue;
+ - (allocations.isEmpty() ? 0 :
+ allocations.get(anyAsk.getCapability()).size());
+ toAllocate = Math.min(toAllocate,
+ appParams.getMaxAllocationsPerSchedulerKeyPerRound());
+ int numAllocated = 0;
+ // Node Candidates are selected as follows:
+ // * Node local candidates selected in loop == 0
+ // * Rack local candidates selected in loop == 1
+ // * From loop == 2 onwards, we revert to off switch allocations.
+ int loopIndex = OFF_SWITCH_LOOP;
+ if (enrichedAsk.getNodeLocations().size() > 0) {
+ loopIndex = NODE_LOCAL_LOOP;
+ }
+ while (numAllocated < toAllocate) {
+ Collection<RemoteNode> nodeCandidates =
+ findNodeCandidates(loopIndex, allNodes, blacklist, enrichedAsk);
+ for (RemoteNode rNode : nodeCandidates) {
+ String rNodeHost = rNode.getNodeId().getHost();
+ // Ignore black list
+ if (blacklist.contains(rNodeHost)) {
+ LOG.info("Nodes for scheduling has a blacklisted node" +
+ " [" + rNodeHost + "]..");
+ continue;
+ }
+ String location = ResourceRequest.ANY;
+ if (loopIndex == NODE_LOCAL_LOOP) {
+ if (enrichedAsk.getNodeLocations().contains(rNodeHost)) {
+ location = rNodeHost;
+ } else {
+ continue;
+ }
+ }
+ if (loopIndex == RACK_LOCAL_LOOP) {
+ if (enrichedAsk.getRackLocations().contains(rNode.getRackName())) {
+ location = rNode.getRackName();
+ } else {
+ continue;
+ }
+ }
+ Container container = createContainer(rmIdentifier, appParams,
+ idCounter, id, userName, allocations, location,
+ anyAsk, rNode);
+ numAllocated++;
+ // Try to spread the allocations across the nodes.
+ // But don't add if it is a node local request.
+ if (loopIndex != NODE_LOCAL_LOOP) {
+ blacklist.add(rNode.getNodeId().getHost());
+ }
+ LOG.info("Allocated [" + container.getId() + "] as opportunistic at " +
+ "location [" + location + "]");
+ if (numAllocated >= toAllocate) {
+ break;
+ }
+ }
+ if (loopIndex == NODE_LOCAL_LOOP &&
+ enrichedAsk.getRackLocations().size() > 0) {
+ loopIndex = RACK_LOCAL_LOOP;
+ } else {
+ loopIndex++;
+ }
+ // Handle case where there are no nodes remaining after blacklist is
+ // considered.
+ if (loopIndex > OFF_SWITCH_LOOP && numAllocated == 0) {
+ LOG.warn("Unable to allocate any opportunistic containers.");
+ break;
}
- nodesForScheduling.add(nodeEntry.getValue());
}
- if (nodesForScheduling.isEmpty()) {
- LOG.warn("No nodes available for allocating opportunistic containers. [" +
- "allNodes={}, blacklist={}]", allNodes, blacklist);
- return;
+ }
+
+ private Collection<RemoteNode> findNodeCandidates(int loopIndex,
+ Map<String, RemoteNode> allNodes, Set<String> blackList,
+ EnrichedResourceRequest enrichedRR) {
+ if (loopIndex > 1) {
+ return allNodes.values();
+ } else {
+ LinkedList<RemoteNode> retList = new LinkedList<>();
+ int numContainers = enrichedRR.getRequest().getNumContainers();
+ while (numContainers > 0) {
+ if (loopIndex == 0) {
+ // Node local candidates
+ numContainers = collectNodeLocalCandidates(
+ allNodes, enrichedRR, retList, numContainers);
+ } else {
+ // Rack local candidates
+ numContainers = collectRackLocalCandidates(
+ allNodes, enrichedRR, retList, blackList, numContainers);
+ }
+ if (numContainers == enrichedRR.getRequest().getNumContainers()) {
+ // If there is no change in numContainers, then there is no point
+ // in looping again.
+ break;
+ }
+ }
+ return retList;
}
- int numAllocated = 0;
- int nextNodeToSchedule = 0;
- for (int numCont = 0; numCont < toAllocate; numCont++) {
- nextNodeToSchedule++;
- nextNodeToSchedule %= nodesForScheduling.size();
- RemoteNode node = nodesForScheduling.get(nextNodeToSchedule);
- Container container = buildContainer(rmIdentifier, appParams, idCounter,
- anyAsk, id, userName, node);
- List<Container> cList = containers.get(anyAsk.getCapability());
- if (cList == null) {
- cList = new ArrayList<>();
- containers.put(anyAsk.getCapability(), cList);
+ }
+
+ private int collectRackLocalCandidates(Map<String, RemoteNode> allNodes,
+ EnrichedResourceRequest enrichedRR, LinkedList<RemoteNode> retList,
+ Set<String> blackList, int numContainers) {
+ for (RemoteNode rNode : allNodes.values()) {
+ if (enrichedRR.getRackLocations().contains(rNode.getRackName())) {
+ if (blackList.contains(rNode.getNodeId().getHost())) {
+ retList.addLast(rNode);
+ } else {
+ retList.addFirst(rNode);
+ numContainers--;
+ }
+ }
+ if (numContainers == 0) {
+ break;
+ }
+ }
+ return numContainers;
+ }
+
+ private int collectNodeLocalCandidates(Map<String, RemoteNode> allNodes,
+ EnrichedResourceRequest enrichedRR, List<RemoteNode> retList,
+ int numContainers) {
+ for (String nodeName : enrichedRR.getNodeLocations()) {
+ RemoteNode remoteNode = allNodes.get(nodeName);
+ if (remoteNode != null) {
+ retList.add(remoteNode);
+ numContainers--;
}
- cList.add(container);
- numAllocated++;
- LOG.info("Allocated [{}] as opportunistic.", container.getId());
+ if (numContainers == 0) {
+ break;
+ }
+ }
+ return numContainers;
+ }
+
+ private Container createContainer(long rmIdentifier,
+ AllocationParams appParams, ContainerIdGenerator idCounter,
+ ApplicationAttemptId id, String userName,
+ Map<Resource, List<Allocation>> allocations, String location,
+ ResourceRequest anyAsk, RemoteNode rNode) throws YarnException {
+ Container container = buildContainer(rmIdentifier, appParams,
+ idCounter, anyAsk, id, userName, rNode);
+ List<Allocation> allocList = allocations.get(anyAsk.getCapability());
+ if (allocList == null) {
+ allocList = new ArrayList<>();
+ allocations.put(anyAsk.getCapability(), allocList);
}
- LOG.info("Allocated {} opportunistic containers.", numAllocated);
+ allocList.add(new Allocation(container, location));
+ return container;
}
private Container buildContainer(long rmIdentifier,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
index 1b1c5b9..246d450 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerContext.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.scheduler;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RemoteNode;
@@ -35,8 +34,10 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.Allocation;
import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.AllocationParams;
import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.ContainerIdGenerator;
+import static org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator.EnrichedResourceRequest;
/**
* This encapsulates application specific information used by the
@@ -53,7 +54,8 @@ public class OpportunisticContainerContext {
new ContainerIdGenerator();
private volatile List<RemoteNode> nodeList = new LinkedList<>();
- private final Map<String, RemoteNode> nodeMap = new LinkedHashMap<>();
+ private final LinkedHashMap<String, RemoteNode> nodeMap =
+ new LinkedHashMap<>();
private final Set<String> blacklist = new HashSet<>();
@@ -61,7 +63,8 @@ public class OpportunisticContainerContext {
// Resource Name (host/rack/any) and capability. This mapping is required
// to match a received Container to an outstanding OPPORTUNISTIC
// ResourceRequest (ask).
- private final TreeMap<SchedulerRequestKey, Map<Resource, ResourceRequest>>
+ private final TreeMap
+ <SchedulerRequestKey, Map<Resource, EnrichedResourceRequest>>
outstandingOpReqs = new TreeMap<>();
public AllocationParams getAppParams() {
@@ -107,7 +110,7 @@ public class OpportunisticContainerContext {
return blacklist;
}
- public TreeMap<SchedulerRequestKey, Map<Resource, ResourceRequest>>
+ public TreeMap<SchedulerRequestKey, Map<Resource, EnrichedResourceRequest>>
getOutstandingOpReqs() {
return outstandingOpReqs;
}
@@ -125,36 +128,32 @@ public class OpportunisticContainerContext {
for (ResourceRequest request : resourceAsks) {
SchedulerRequestKey schedulerKey = SchedulerRequestKey.create(request);
- // TODO: Extend for Node/Rack locality. We only handle ANY requests now
- if (!ResourceRequest.isAnyLocation(request.getResourceName())) {
- continue;
- }
-
- if (request.getNumContainers() == 0) {
- continue;
- }
-
- Map<Resource, ResourceRequest> reqMap =
+ Map<Resource, EnrichedResourceRequest> reqMap =
outstandingOpReqs.get(schedulerKey);
if (reqMap == null) {
reqMap = new HashMap<>();
outstandingOpReqs.put(schedulerKey, reqMap);
}
- ResourceRequest resourceRequest = reqMap.get(request.getCapability());
- if (resourceRequest == null) {
- resourceRequest = request;
- reqMap.put(request.getCapability(), request);
+ EnrichedResourceRequest eReq = reqMap.get(request.getCapability());
+ if (eReq == null) {
+ eReq = new EnrichedResourceRequest(request);
+ reqMap.put(request.getCapability(), eReq);
+ }
+ // Set numContainers only for ANY request
+ if (ResourceRequest.isAnyLocation(request.getResourceName())) {
+ eReq.getRequest().setResourceName(ResourceRequest.ANY);
+ eReq.getRequest().setNumContainers(request.getNumContainers());
} else {
- resourceRequest.setNumContainers(
- resourceRequest.getNumContainers() + request.getNumContainers());
+ eReq.addLocation(request.getResourceName(), request.getNumContainers());
}
if (ResourceRequest.isAnyLocation(request.getResourceName())) {
LOG.info("# of outstandingOpReqs in ANY (at "
+ "priority = " + schedulerKey.getPriority()
+ ", allocationReqId = " + schedulerKey.getAllocationRequestId()
+ ", with capability = " + request.getCapability() + " ) : "
- + resourceRequest.getNumContainers());
+ + ", with location = " + request.getResourceName() + " ) : "
+ + ", numContainers = " + eReq.getRequest().getNumContainers());
}
}
}
@@ -163,25 +162,34 @@ public class OpportunisticContainerContext {
* This method matches a returned list of Container Allocations to any
* outstanding OPPORTUNISTIC ResourceRequest.
* @param capability Capability
- * @param allocatedContainers Allocated Containers
+ * @param allocations Allocations.
*/
public void matchAllocationToOutstandingRequest(Resource capability,
- List<Container> allocatedContainers) {
- for (Container c : allocatedContainers) {
+ List<Allocation> allocations) {
+ for (OpportunisticContainerAllocator.Allocation allocation : allocations) {
SchedulerRequestKey schedulerKey =
- SchedulerRequestKey.extractFrom(c);
- Map<Resource, ResourceRequest> asks =
+ SchedulerRequestKey.extractFrom(allocation.getContainer());
+ Map<Resource, EnrichedResourceRequest> asks =
outstandingOpReqs.get(schedulerKey);
if (asks == null) {
continue;
}
- ResourceRequest rr = asks.get(capability);
- if (rr != null) {
- rr.setNumContainers(rr.getNumContainers() - 1);
- if (rr.getNumContainers() == 0) {
+ EnrichedResourceRequest err = asks.get(capability);
+ if (err != null) {
+ int numContainers = err.getRequest().getNumContainers();
+ numContainers--;
+ err.getRequest().setNumContainers(numContainers);
+ if (numContainers == 0) {
asks.remove(capability);
+ if (asks.size() == 0) {
+ outstandingOpReqs.remove(schedulerKey);
+ }
+ } else {
+ if (!ResourceRequest.isAnyLocation(allocation.getResourceName())) {
+ err.removeLocation(allocation.getResourceName());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index e889cde..8e59f14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -30,6 +30,7 @@ import "yarn_service_protos.proto";
message RemoteNodeProto {
optional NodeIdProto node_id = 1;
optional string http_address = 2;
+ optional string rack_name = 3;
}
message RegisterDistributedSchedulingAMResponseProto {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-7009.
TestNMClient.testNMClientNoCleanupOnStop is flaky by design. (Miklos Szegedi
via Haibo Chen)
Posted by jh...@apache.org.
YARN-7009. TestNMClient.testNMClientNoCleanupOnStop is flaky by design. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c071aad5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c071aad5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c071aad5
Branch: refs/heads/YARN-5734
Commit: c071aad5da6f6601978e73d38ce5813958848bc4
Parents: b6942cb
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Oct 5 13:17:32 2017 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Oct 5 15:50:13 2017 -0700
----------------------------------------------------------------------
.../src/main/data/2jobs2min-rumen-jh.json | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../yarn/client/api/impl/TestNMClient.java | 265 ++++++++++++++-----
.../src/main/resources/yarn-default.xml | 2 +-
.../nodemanager/DefaultContainerExecutor.java | 2 +-
.../launcher/ContainerLaunch.java | 20 ++
6 files changed, 221 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
index 8298d60..c096229 100644
--- a/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
+++ b/hadoop-tools/hadoop-sls/src/main/data/2jobs2min-rumen-jh.json
@@ -4869,7 +4869,7 @@
"hadoop.security.groups.cache.secs" : "300",
"ipc.client.connect.max.retries" : "10",
"dfs.namenode.delegation.key.update-interval" : "86400000",
- "yarn.nodemanager.process-kill-wait.ms" : "2000",
+ "yarn.nodemanager.process-kill-wait.ms" : "5000",
"yarn.application.classpath" : "$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$YARN_HOME/*,$YARN_HOME/lib/*",
"yarn.app.mapreduce.client.max-retries" : "3",
"dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction" : "0.75f",
@@ -9950,7 +9950,7 @@
"hadoop.security.groups.cache.secs" : "300",
"ipc.client.connect.max.retries" : "10",
"dfs.namenode.delegation.key.update-interval" : "86400000",
- "yarn.nodemanager.process-kill-wait.ms" : "2000",
+ "yarn.nodemanager.process-kill-wait.ms" : "5000",
"yarn.application.classpath" : "$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$YARN_HOME/*,$YARN_HOME/lib/*",
"yarn.app.mapreduce.client.max-retries" : "3",
"dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction" : "0.75f",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f58833c..d99941d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1907,7 +1907,7 @@ public class YarnConfiguration extends Configuration {
public static final String NM_PROCESS_KILL_WAIT_MS =
NM_PREFIX + "process-kill-wait.ms";
public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
- 2000;
+ 5000;
/** Max time to wait to establish a connection to RM */
public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index 016f1bc..8780d9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -27,10 +27,15 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
@@ -67,6 +72,10 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerStateTransitionListener;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
@@ -85,11 +94,78 @@ public class TestNMClient {
ApplicationAttemptId attemptId = null;
int nodeCount = 3;
NMTokenCache nmTokenCache = null;
-
+
+ /**
+ * Container State transition listener to track the number of times
+ * a container has transitioned into a state.
+ */
+ public static class DebugSumContainerStateListener
+ implements ContainerStateTransitionListener {
+
+ private static final Log LOG =
+ LogFactory.getLog(DebugSumContainerStateListener.class);
+ private static final Map<ContainerId,
+ Map<org.apache.hadoop.yarn.server.nodemanager.containermanager
+ .container.ContainerState, Long>>
+ TRANSITION_COUNTER = new HashMap<>();
+
+ public void init(Context context) {
+ }
+
+ public void preTransition(ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager
+ .containermanager.container.ContainerState
+ beforeState,
+ ContainerEvent eventToBeProcessed) {
+ }
+
+ public void postTransition(
+ ContainerImpl op,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState beforeState,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState afterState,
+ ContainerEvent processedEvent) {
+ synchronized (TRANSITION_COUNTER) {
+ if (beforeState != afterState) {
+ ContainerId id = op.getContainerId();
+ TRANSITION_COUNTER
+ .putIfAbsent(id, new HashMap<>());
+ long sum = TRANSITION_COUNTER.get(id)
+ .compute(afterState,
+ (state, count) -> count == null ? 1 : count + 1);
+ LOG.info("***** " + id +
+ " Transition from " + beforeState +
+ " to " + afterState +
+ "sum:" + sum);
+ }
+ }
+ }
+
+ /**
+ * Get the current number of state transitions.
+ * This is useful to check, if an event has occurred in unit tests.
+ * @param id Container id to check
+ * @param state Return the overall number of transitions to this state
+ * @return Number of transitions to the state specified
+ */
+ static long getTransitionCounter(ContainerId id,
+ org.apache.hadoop.yarn.server.nodemanager
+ .containermanager.container
+ .ContainerState state) {
+ Long ret = TRANSITION_COUNTER.getOrDefault(id, new HashMap<>())
+ .get(state);
+ return ret != null ? ret : 0;
+ }
+ }
+
@Before
public void setup() throws YarnException, IOException {
// start minicluster
conf = new YarnConfiguration();
+ // Turn on state tracking
+ conf.set(YarnConfiguration.NM_CONTAINER_STATE_TRANSITION_LISTENERS,
+ DebugSumContainerStateListener.class.getName());
yarnCluster =
new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
yarnCluster.init(conf);
@@ -290,7 +366,7 @@ public class TestNMClient {
return containers;
}
- private void testContainerManagement(NMClientImpl nmClient,
+ private void testContainerManagement(NMClientImpl client,
Set<Container> containers) throws YarnException, IOException {
int size = containers.size();
int i = 0;
@@ -298,7 +374,7 @@ public class TestNMClient {
// getContainerStatus shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.getContainerStatus(container.getId(), container.getNodeId());
+ client.getContainerStatus(container.getId(), container.getNodeId());
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -307,7 +383,7 @@ public class TestNMClient {
// upadateContainerResource shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.updateContainerResource(container);
+ client.updateContainerResource(container);
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -317,7 +393,7 @@ public class TestNMClient {
// restart shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.restartContainer(container.getId());
+ client.restartContainer(container.getId());
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -327,7 +403,7 @@ public class TestNMClient {
// rollback shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.rollbackLastReInitialization(container.getId());
+ client.rollbackLastReInitialization(container.getId());
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -337,7 +413,7 @@ public class TestNMClient {
// commit shouldn't be called before startContainer,
// otherwise, NodeManager cannot find the container
try {
- nmClient.commitLastReInitialization(container.getId());
+ client.commitLastReInitialization(container.getId());
fail("Exception is expected");
} catch (YarnException e) {
assertTrue("The thrown exception is not expected",
@@ -347,14 +423,12 @@ public class TestNMClient {
// stopContainer shouldn't be called before startContainer,
// otherwise, an exception will be thrown
try {
- nmClient.stopContainer(container.getId(), container.getNodeId());
+ client.stopContainer(container.getId(), container.getNodeId());
fail("Exception is expected");
} catch (YarnException e) {
if (!e.getMessage()
.contains("is not handled by this NodeManager")) {
- throw (AssertionError)
- (new AssertionError("Exception is not expected: " + e).initCause(
- e));
+ throw new AssertionError("Exception is not expected: ", e);
}
}
@@ -367,78 +441,131 @@ public class TestNMClient {
Records.newRecord(ContainerLaunchContext.class);
if (Shell.WINDOWS) {
clc.setCommands(
- Arrays.asList("ping", "-n", "100", "127.0.0.1", ">nul"));
+ Arrays.asList("ping", "-n", "10000000", "127.0.0.1", ">nul"));
} else {
- clc.setCommands(Arrays.asList("sleep", "10"));
+ clc.setCommands(Arrays.asList("sleep", "1000000"));
}
clc.setTokens(securityTokens);
try {
- nmClient.startContainer(container, clc);
+ client.startContainer(container, clc);
} catch (YarnException e) {
- throw (AssertionError)
- (new AssertionError("Exception is not expected: " + e).initCause(e));
+ throw new AssertionError("Exception is not expected ", e);
}
+ List<Integer> exitStatuses = Collections.singletonList(-1000);
// leave one container unclosed
if (++i < size) {
- // NodeManager may still need some time to make the container started
- testGetContainerStatus(container, i, ContainerState.RUNNING, "",
- Arrays.asList(new Integer[] {-1000}));
- // Test increase container API and make sure requests can reach NM
- testIncreaseContainerResource(container);
-
- testRestartContainer(container.getId());
- testGetContainerStatus(container, i, ContainerState.RUNNING,
- "will be Restarted", Arrays.asList(new Integer[] {-1000}));
-
- if (i % 2 == 0) {
- testReInitializeContainer(container.getId(), clc, false);
- testGetContainerStatus(container, i, ContainerState.RUNNING,
- "will be Re-initialized", Arrays.asList(new Integer[] {-1000}));
- testRollbackContainer(container.getId(), false);
- testGetContainerStatus(container, i, ContainerState.RUNNING,
- "will be Rolled-back", Arrays.asList(new Integer[] {-1000}));
- testCommitContainer(container.getId(), true);
- testReInitializeContainer(container.getId(), clc, false);
- testGetContainerStatus(container, i, ContainerState.RUNNING,
- "will be Re-initialized", Arrays.asList(new Integer[] {-1000}));
- testCommitContainer(container.getId(), false);
- } else {
- testReInitializeContainer(container.getId(), clc, true);
- testGetContainerStatus(container, i, ContainerState.RUNNING,
- "will be Re-initialized", Arrays.asList(new Integer[] {-1000}));
- testRollbackContainer(container.getId(), true);
- testCommitContainer(container.getId(), true);
- }
+ testContainer(client, i, container, clc, exitStatuses);
- try {
- nmClient.stopContainer(container.getId(), container.getNodeId());
- } catch (YarnException e) {
- throw (AssertionError)
- (new AssertionError("Exception is not expected: " + e)
- .initCause(e));
- }
+ }
+ }
+ }
+
+ private void testContainer(NMClientImpl client, int i, Container container,
+ ContainerLaunchContext clc, List<Integer> exitCode)
+ throws YarnException, IOException {
+ // NodeManager may still need some time to make the container started
+ testGetContainerStatus(container, i, ContainerState.RUNNING, "",
+ exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 1);
+ // Test increase container API and make sure requests can reach NM
+ testIncreaseContainerResource(container);
+
+ testRestartContainer(container.getId());
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Restarted", exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 2);
+
+ if (i % 2 == 0) {
+ testReInitializeContainer(container.getId(), clc, false);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Re-initialized", exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 3);
+
+ testRollbackContainer(container.getId(), false);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Rolled-back", exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 4);
+
+ testCommitContainer(container.getId(), true);
+ testReInitializeContainer(container.getId(), clc, false);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Re-initialized", exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 5);
+ testCommitContainer(container.getId(), false);
+ } else {
+ testReInitializeContainer(container.getId(), clc, true);
+ testGetContainerStatus(container, i, ContainerState.RUNNING,
+ "will be Re-initialized", exitCode);
+ waitForContainerTransitionCount(container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING, 3);
+ testRollbackContainer(container.getId(), true);
+ testCommitContainer(container.getId(), true);
+ }
+
+ try {
+ client.stopContainer(container.getId(), container.getNodeId());
+ } catch (YarnException e) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e, e));
+ }
- // getContainerStatus can be called after stopContainer
+ // getContainerStatus can be called after stopContainer
+ try {
+ // O is possible if CLEANUP_CONTAINER is executed too late
+ // -105 is possible if the container is not terminated but killed
+ testGetContainerStatus(container, i, ContainerState.COMPLETE,
+ "Container killed by the ApplicationMaster.",
+ Arrays.asList(
+ ContainerExitStatus.KILLED_BY_APPMASTER,
+ ContainerExitStatus.SUCCESS));
+ } catch (YarnException e) {
+ // The exception is possible because, after the container is stopped,
+ // it may be removed from NM's context.
+ if (!e.getMessage()
+ .contains("was recently stopped on node manager")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: ", e));
+ }
+ }
+ }
+
+ /**
+ * Wait until the container reaches a state N times.
+ * @param container container to watch
+ * @param state state to test
+ * @param transitions the number N above
+ * @throws YarnException This happens if the test times out while waiting
+ */
+ private void waitForContainerTransitionCount(
+ Container container,
+ org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState state, long transitions)
+ throws YarnException {
+ long transitionCount = -1;
+ do {
+ if (transitionCount != -1) {
try {
- // O is possible if CLEANUP_CONTAINER is executed too late
- // -105 is possible if the container is not terminated but killed
- testGetContainerStatus(container, i, ContainerState.COMPLETE,
- "Container killed by the ApplicationMaster.", Arrays.asList(
- new Integer[] {ContainerExitStatus.KILLED_BY_APPMASTER,
- ContainerExitStatus.SUCCESS}));
- } catch (YarnException e) {
- // The exception is possible because, after the container is stopped,
- // it may be removed from NM's context.
- if (!e.getMessage()
- .contains("was recently stopped on node manager")) {
- throw (AssertionError)
- (new AssertionError("Exception is not expected: " + e).initCause(
- e));
- }
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ throw new YarnException(
+ "Timeout at transition count:" + transitionCount, e);
}
}
- }
+ transitionCount = DebugSumContainerStateListener
+ .getTransitionCounter(container.getId(), state);
+ } while (transitionCount != transitions);
}
private void sleep(int sleepTime) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8b22fa4..2bc2330 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1747,7 +1747,7 @@
<property>
<description>Max time to wait for a process to come up when trying to cleanup a container</description>
<name>yarn.nodemanager.process-kill-wait.ms</name>
- <value>2000</value>
+ <value>5000</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 2b32234..7c11d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -275,7 +275,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
sb.writeLocalWrapperScript(launchDst, pidFile);
} else {
LOG.info("Container " + containerIdStr
- + " was marked as inactive. Returning terminated error");
+ + " pid file not set. Returning terminated error");
return ExitCode.TERMINATED.getExitCode();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c071aad5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f67dce7..f1c826e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -734,6 +734,26 @@ public class ContainerLaunch implements Callable<Integer> {
new DelayedProcessKiller(container, user,
processId, sleepDelayBeforeSigKill, Signal.KILL, exec).start();
}
+ } else {
+ // Normally this means that the process was notified about
+ // deactivateContainer above and did not start.
+ // Since we already set the state to RUNNING or REINITIALIZING
+ // we have to send a killed event to continue.
+ if (!completed.get()) {
+ LOG.warn("Container clean up before pid file created "
+ + containerIdStr);
+ dispatcher.getEventHandler().handle(
+ new ContainerExitEvent(container.getContainerId(),
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
+ ExitCode.TERMINATED.getExitCode(),
+ "Container terminated before pid file created."));
+ // There is a possibility that the launch grabbed the file name before
+ // the deactivateContainer above but it was slow enough to avoid
+ // getContainerPid.
+ // Increasing YarnConfiguration.NM_PROCESS_KILL_WAIT_MS
+ // reduces the likelihood of this race condition and process leak.
+ }
}
} catch (Exception e) {
String message =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDFS-11968. ViewFS:
StoragePolicies commands fail with HDFS federation. Contributed by Mukul
Kumar Singh.
Posted by jh...@apache.org.
HDFS-11968. ViewFS: StoragePolicies commands fail with HDFS federation. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9130511
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9130511
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9130511
Branch: refs/heads/YARN-5734
Commit: b91305119b434d23b99ae7e755aea6639f48b6ab
Parents: 4d5dd75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Oct 3 11:23:40 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Oct 3 11:23:40 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/viewfs/InodeTree.java | 3 +-
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 +++++++++------
.../hdfs/tools/TestStoragePolicyCommands.java | 11 +--
.../tools/TestViewFSStoragePolicyCommands.java | 80 ++++++++++++++++++++
.../tools/TestWebHDFSStoragePolicyCommands.java | 42 ++++++++++
5 files changed, 168 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index 665c9c9..7274acd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -407,7 +407,8 @@ abstract class InodeTree<T> {
for (int j = 1; j <= i; ++j) {
failedAt.append('/').append(path[j]);
}
- throw (new FileNotFoundException(failedAt.toString()));
+ throw (new FileNotFoundException("File/Directory does not exist: "
+ + failedAt.toString()));
}
if (nextInode instanceof INodeLink) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
index 9c7d048..d5e5b4d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hdfs.tools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.BlockStoragePolicySpi;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -29,6 +31,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -100,11 +103,12 @@ public class StoragePolicyAdmin extends Configured implements Tool {
@Override
public int run(Configuration conf, List<String> args) throws IOException {
- final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
System.out.println("Block Storage Policies:");
- for (BlockStoragePolicy policy : policies) {
+ for (BlockStoragePolicySpi policy : policies) {
if (policy != null) {
System.out.println("\t" + policy);
}
@@ -149,32 +153,43 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- HdfsFileStatus status = dfs.getClient().getFileInfo(
- Path.getPathWithoutSchemeAndAuthority(p).toString());
- if (status == null) {
+ FileStatus status;
+ try {
+ status = fs.getFileStatus(p);
+ } catch (FileNotFoundException e) {
System.err.println("File/Directory does not exist: " + path);
return 2;
}
- byte storagePolicyId = status.getStoragePolicy();
- if (storagePolicyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
- System.out.println("The storage policy of " + path + " is unspecified");
- return 0;
- }
- Collection<BlockStoragePolicy> policies = dfs.getAllStoragePolicies();
- for (BlockStoragePolicy policy : policies) {
- if (policy.getId() == storagePolicyId) {
- System.out.println("The storage policy of " + path + ":\n" + policy);
+
+ if (status instanceof HdfsFileStatus) {
+ byte storagePolicyId = ((HdfsFileStatus)status).getStoragePolicy();
+ if (storagePolicyId ==
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
+ System.out.println("The storage policy of " + path
+ + " is unspecified");
return 0;
}
+ Collection<? extends BlockStoragePolicySpi> policies =
+ fs.getAllStoragePolicies();
+ for (BlockStoragePolicySpi policy : policies) {
+ if (policy instanceof BlockStoragePolicy) {
+ if (((BlockStoragePolicy)policy).getId() == storagePolicyId) {
+ System.out.println("The storage policy of " + path
+ + ":\n" + policy);
+ return 0;
+ }
+ }
+ }
}
+ System.err.println(getName() + " is not supported for filesystem "
+ + fs.getScheme() + " on path " + path);
+ return 2;
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
return 2;
}
- System.err.println("Cannot identify the storage policy for " + path);
- return 2;
}
}
@@ -218,9 +233,9 @@ public class StoragePolicyAdmin extends Configured implements Tool {
return 1;
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- dfs.setStoragePolicy(p, policyName);
+ fs.setStoragePolicy(p, policyName);
System.out.println("Set storage policy " + policyName + " on " + path);
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
@@ -264,9 +279,9 @@ public class StoragePolicyAdmin extends Configured implements Tool {
}
Path p = new Path(path);
- final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
+ final FileSystem fs = FileSystem.get(conf);
try {
- dfs.unsetStoragePolicy(p);
+ fs.unsetStoragePolicy(p);
System.out.println("Unset storage policy from " + path);
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
index 149dabb..f31c739 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java
@@ -18,11 +18,12 @@
package org.apache.hadoop.hdfs.tools;
import java.io.IOException;
+import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -38,12 +39,12 @@ public class TestStoragePolicyCommands {
private static final short REPL = 1;
private static final int SIZE = 128;
- private static Configuration conf;
- private static MiniDFSCluster cluster;
- private static DistributedFileSystem fs;
+ protected static Configuration conf;
+ protected static MiniDFSCluster cluster;
+ protected static FileSystem fs;
@Before
- public void clusterSetUp() throws IOException {
+ public void clusterSetUp() throws IOException, URISyntaxException {
conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build();
cluster.waitActive();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
new file mode 100644
index 0000000..b3bb3c4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FsConstants;
+
+import org.apache.hadoop.fs.viewfs.ConfigUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Test StoragePolicyAdmin commands with ViewFileSystem.
+ */
+public class TestViewFSStoragePolicyCommands extends TestStoragePolicyCommands {
+
+ @Before
+ public void clusterSetUp() throws IOException {
+ conf = new HdfsConfiguration();
+ String clusterName = "cluster";
+ cluster =
+ new MiniDFSCluster.Builder(conf).nnTopology(
+ MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem hdfs1 = cluster.getFileSystem(0);
+ DistributedFileSystem hdfs2 = cluster.getFileSystem(1);
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ FsConstants.VIEWFS_SCHEME +"://" + clusterName);
+
+ Path base1 = new Path("/user1");
+ Path base2 = new Path("/user2");
+ hdfs1.delete(base1, true);
+ hdfs2.delete(base2, true);
+ hdfs1.mkdirs(base1);
+ hdfs2.mkdirs(base2);
+ ConfigUtil.addLink(conf, clusterName, "/foo",
+ hdfs1.makeQualified(base1).toUri());
+ ConfigUtil.addLink(conf, clusterName, "/hdfs2",
+ hdfs2.makeQualified(base2).toUri());
+ fs = FileSystem.get(conf);
+ }
+
+ /**
+ * Storage policy operation on the viewfs root should fail.
+ */
+ @Test
+ public void testStoragePolicyRoot() throws Exception {
+ final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
+ DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /", 2,
+ "is not supported for filesystem viewfs on path /");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9130511/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
new file mode 100644
index 0000000..f10205c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * Test StoragePolicyAdmin commands with WebHDFS.
+ */
+public class TestWebHDFSStoragePolicyCommands
+ extends TestStoragePolicyCommands {
+
+ @Before
+ public void clusterSetUp() throws IOException, URISyntaxException {
+ super.clusterSetUp();
+ fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsConstants.WEBHDFS_SCHEME);
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
+ fs.getUri().toString());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-5329. Placement Agent
enhancements required to support recurring reservations in ReservationSystem.
(Carlo Curino via Subru).
Posted by jh...@apache.org.
YARN-5329. Placement Agent enhancements required to support recurring reservations in ReservationSystem. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6e614e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6e614e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6e614e3
Branch: refs/heads/YARN-5734
Commit: e6e614e380ed1d746973b50f666a9c40d272073e
Parents: fe84cdc
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Oct 4 19:28:27 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Oct 4 19:28:27 2017 -0700
----------------------------------------------------------------------
.../reservation/InMemoryPlan.java | 6 +
.../PeriodicRLESparseResourceAllocation.java | 3 +-
.../RLESparseResourceAllocation.java | 6 +-
.../reservation/planning/IterativePlanner.java | 27 ++-
.../reservation/planning/PlanningAlgorithm.java | 79 ++++---
.../reservation/planning/StageAllocator.java | 3 +-
.../planning/StageAllocatorGreedy.java | 4 +-
.../planning/StageAllocatorGreedyRLE.java | 7 +-
.../planning/StageAllocatorLowCostAligned.java | 6 +-
.../reservation/BaseSharingPolicyTest.java | 8 +-
.../reservation/TestCapacityOverTimePolicy.java | 29 +--
.../reservation/TestInMemoryPlan.java | 69 +++---
.../planning/TestAlignedPlanner.java | 40 +++-
.../planning/TestGreedyReservationAgent.java | 69 +++---
.../planning/TestReservationAgents.java | 213 +++++++++++++++++++
15 files changed, 442 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
index 9eb1820..7187510 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java
@@ -723,6 +723,12 @@ public class InMemoryPlan implements Plan {
+ periodicRle.getTimePeriod() + ")");
}
+ if (period < (end - start)) {
+ throw new PlanningException(
+ "Invalid input: (end - start) = (" + end + " - " + start + ") = "
+ + (end - start) + " > period = " + period);
+ }
+
// find the minimum resources available among all the instances that fit
// in the LCM
long numInstInLCM = periodicRle.getTimePeriod() / period;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
index 7bc44f5..d326944 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/PeriodicRLESparseResourceAllocation.java
@@ -221,7 +221,8 @@ public class PeriodicRLESparseResourceAllocation
NavigableMap<Long, Resource> cumulativeMap = this.getCumulative();
Long previous = cumulativeMap.floorKey(relativeStart);
previous = (previous != null) ? previous : 0;
- for (long i = 0; i <= (end - start) / timePeriod; i++) {
+ //make sure to go one past end, to catch end times extending past period
+ for (long i = 0; i <= 1 + (end - start) / timePeriod; i++) {
for (Map.Entry<Long, Resource> e : cumulativeMap.entrySet()) {
long curKey = e.getKey() + (i * timePeriod);
if (curKey >= previous && (start + curKey - relativeStart) <= end) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
index 3062f3d..8280bc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/RLESparseResourceAllocation.java
@@ -423,7 +423,8 @@ public class RLESparseResourceAllocation {
Resource outRes) {
if (out.isEmpty() || (out.lastEntry() != null && outRes == null)
- || !Resources.equals(out.lastEntry().getValue(), outRes)) {
+ || (out.lastEntry().getValue() != null
+ && !Resources.equals(out.lastEntry().getValue(), outRes))) {
out.put(time, outRes);
}
@@ -460,7 +461,8 @@ public class RLESparseResourceAllocation {
if (!Resources.fitsIn(b, a)) {
throw new PlanningException(
"RLESparseResourceAllocation: merge failed as the "
- + "resulting RLESparseResourceAllocation would be negative");
+ + "resulting RLESparseResourceAllocation would "
+ + "be negative, when testing: (" + eB + ") > (" + eA + ")");
} else {
return Resources.subtract(a, b);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/IterativePlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/IterativePlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/IterativePlanner.java
index 83f272e..9b18e90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/IterativePlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/IterativePlanner.java
@@ -98,6 +98,12 @@ public class IterativePlanner extends PlanningAlgorithm {
// Current stage
ReservationRequest currentReservationStage;
+ // initialize periodicity
+ long period = 0;
+ if(reservation.getRecurrenceExpression() != null){
+ period = Long.parseLong(reservation.getRecurrenceExpression());
+ }
+
// Iterate the stages in reverse order
while (stageProvider.hasNext()) {
@@ -117,7 +123,7 @@ public class IterativePlanner extends PlanningAlgorithm {
// Compute stage allocation
Map<ReservationInterval, Resource> curAlloc =
computeStageAllocation(plan, currentReservationStage, stageArrival,
- stageDeadline, user, reservationId);
+ stageDeadline, period, user, reservationId);
// If we did not find an allocation, return NULL
// (unless it's an ANY job, then we simply continue).
@@ -216,11 +222,10 @@ public class IterativePlanner extends PlanningAlgorithm {
planLoads = plan.getCumulativeLoadOverTime(jobArrival, jobDeadline);
ReservationAllocation oldRes = plan.getReservationById(reservationId);
if (oldRes != null) {
- planLoads =
- RLESparseResourceAllocation.merge(plan.getResourceCalculator(),
- plan.getTotalCapacity(), planLoads,
- oldRes.getResourcesOverTime(), RLEOperator.subtract, jobArrival,
- jobDeadline);
+ planLoads = RLESparseResourceAllocation.merge(
+ plan.getResourceCalculator(), plan.getTotalCapacity(), planLoads,
+ oldRes.getResourcesOverTime(jobArrival, jobDeadline),
+ RLEOperator.subtract, jobArrival, jobDeadline);
}
}
@@ -309,13 +314,13 @@ public class IterativePlanner extends PlanningAlgorithm {
}
// Call algStageAllocator
- protected Map<ReservationInterval, Resource> computeStageAllocation(
- Plan plan, ReservationRequest rr, long stageArrivalTime,
- long stageDeadline, String user, ReservationId oldId)
- throws PlanningException {
+ protected Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
+ ReservationRequest rr, long stageArrivalTime, long stageDeadline,
+ long period, String user, ReservationId oldId) throws PlanningException {
return algStageAllocator.computeStageAllocation(plan, planLoads,
- planModifications, rr, stageArrivalTime, stageDeadline, user, oldId);
+ planModifications, rr, stageArrivalTime, stageDeadline, period, user,
+ oldId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index bbbf0d6..d4b4b9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.PeriodicRLESparseResourceAllocation;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
@@ -63,21 +64,33 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
// Compute the job allocation
RLESparseResourceAllocation allocation =
- computeJobAllocation(plan, reservationId, adjustedContract, user);
+ computeJobAllocation(plan, reservationId, adjustedContract, user);
+
+ long period = Long.parseLong(contract.getRecurrenceExpression());
+
+ // Make allocation periodic if request is periodic
+ if (contract.getRecurrenceExpression() != null) {
+ if (period > 0) {
+ allocation =
+ new PeriodicRLESparseResourceAllocation(allocation, period);
+ }
+ }
// If no job allocation was found, fail
if (allocation == null) {
throw new PlanningException(
- "The planning algorithm could not find a valid allocation"
- + " for your request");
+ "The planning algorithm could not find a valid allocation"
+ + " for your request");
}
// Translate the allocation to a map (with zero paddings)
long step = plan.getStep();
+
long jobArrival = stepRoundUp(adjustedContract.getArrival(), step);
long jobDeadline = stepRoundUp(adjustedContract.getDeadline(), step);
+
Map<ReservationInterval, Resource> mapAllocations =
- allocationsToPaddedMap(allocation, jobArrival, jobDeadline);
+ allocationsToPaddedMap(allocation, jobArrival, jobDeadline, period);
// Create the reservation
ReservationAllocation capReservation =
@@ -85,8 +98,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
adjustedContract, // Contract
user, // User name
plan.getQueueName(), // Queue name
- findEarliestTime(mapAllocations), // Earliest start time
- findLatestTime(mapAllocations), // Latest end time
+ adjustedContract.getArrival(), adjustedContract.getDeadline(),
mapAllocations, // Allocations
plan.getResourceCalculator(), // Resource calculator
plan.getMinimumAllocation()); // Minimum allocation
@@ -100,33 +112,46 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
}
- private Map<ReservationInterval, Resource>
- allocationsToPaddedMap(RLESparseResourceAllocation allocation,
- long jobArrival, long jobDeadline) {
-
- // Allocate
- Map<ReservationInterval, Resource> mapAllocations =
- allocation.toIntervalMap();
+ private Map<ReservationInterval, Resource> allocationsToPaddedMap(
+ RLESparseResourceAllocation allocation, long jobArrival, long jobDeadline,
+ long period) {
// Zero allocation
Resource zeroResource = Resource.newInstance(0, 0);
- // Pad at the beginning
- long earliestStart = findEarliestTime(mapAllocations);
- if (jobArrival < earliestStart) {
- mapAllocations.put(new ReservationInterval(jobArrival, earliestStart),
- zeroResource);
- }
+ if (period > 0) {
+ if ((jobDeadline - jobArrival) >= period) {
+ allocation.addInterval(new ReservationInterval(0L, period),
+ zeroResource);
+ }
+ jobArrival = jobArrival % period;
+ jobDeadline = jobDeadline % period;
+
+ if (jobArrival <= jobDeadline) {
+ allocation.addInterval(new ReservationInterval(0, jobArrival),
+ zeroResource);
+ allocation.addInterval(new ReservationInterval(jobDeadline, period),
+ zeroResource);
+ } else {
+ allocation.addInterval(new ReservationInterval(jobDeadline, jobArrival),
+ zeroResource);
+ }
+ } else {
+ // Pad at the beginning
+ long earliestStart = findEarliestTime(allocation.toIntervalMap());
+ if (jobArrival < earliestStart) {
+ allocation.addInterval(
+ new ReservationInterval(jobArrival, earliestStart), zeroResource);
+ }
- // Pad at the beginning
- long latestEnd = findLatestTime(mapAllocations);
- if (latestEnd < jobDeadline) {
- mapAllocations.put(new ReservationInterval(latestEnd, jobDeadline),
- zeroResource);
+ // Pad at the beginning
+ long latestEnd = findLatestTime(allocation.toIntervalMap());
+ if (latestEnd < jobDeadline) {
+ allocation.addInterval(new ReservationInterval(latestEnd, jobDeadline),
+ zeroResource);
+ }
}
-
- return mapAllocations;
-
+ return allocation.toIntervalMap();
}
public abstract RLESparseResourceAllocation computeJobAllocation(Plan plan,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
index 8934b0f..1b47a69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocator.java
@@ -45,6 +45,7 @@ public interface StageAllocator {
* the stage by the two phase planning algorithm
* @param stageDeadline the deadline of the stage set by the two phase
* planning algorithm
+ * @param period the periodicity with which this stage appears
* @param user name of the user
* @param oldId identifier of the old reservation
*
@@ -55,7 +56,7 @@ public interface StageAllocator {
Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
RLESparseResourceAllocation planModifications, ReservationRequest rr,
- long stageArrival, long stageDeadline, String user,
+ long stageArrival, long stageDeadline, long period, String user,
ReservationId oldId) throws PlanningException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
index d107487..267e673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedy.java
@@ -43,7 +43,7 @@ public class StageAllocatorGreedy implements StageAllocator {
public Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
RLESparseResourceAllocation planModifications, ReservationRequest rr,
- long stageEarliestStart, long stageDeadline, String user,
+ long stageEarliestStart, long stageDeadline, long period, String user,
ReservationId oldId) throws PlanningException {
Resource totalCapacity = plan.getTotalCapacity();
@@ -69,7 +69,7 @@ public class StageAllocatorGreedy implements StageAllocator {
RLESparseResourceAllocation netAvailable =
plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
- stageDeadline, 0);
+ stageDeadline, period);
netAvailable =
RLESparseResourceAllocation.merge(plan.getResourceCalculator(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
index ae7d91a..a11ea0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorGreedyRLE.java
@@ -54,7 +54,7 @@ public class StageAllocatorGreedyRLE implements StageAllocator {
public Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
RLESparseResourceAllocation planModifications, ReservationRequest rr,
- long stageEarliestStart, long stageDeadline, String user,
+ long stageEarliestStart, long stageDeadline, long period, String user,
ReservationId oldId) throws PlanningException {
// abort early if the interval is not satisfiable
@@ -83,8 +83,9 @@ public class StageAllocatorGreedyRLE implements StageAllocator {
int gangsToPlace = rr.getNumContainers() / rr.getConcurrency();
// get available resources from plan
- RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
- user, oldId, stageEarliestStart, stageDeadline, 0);
+ RLESparseResourceAllocation netRLERes =
+ plan.getAvailableResourceOverTime(user, oldId, stageEarliestStart,
+ stageDeadline, period);
// remove plan modifications
netRLERes =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
index c014549..f67973f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/StageAllocatorLowCostAligned.java
@@ -70,15 +70,15 @@ public class StageAllocatorLowCostAligned implements StageAllocator {
public Map<ReservationInterval, Resource> computeStageAllocation(Plan plan,
RLESparseResourceAllocation planLoads,
RLESparseResourceAllocation planModifications, ReservationRequest rr,
- long stageArrival, long stageDeadline, String user, ReservationId oldId)
- throws PlanningException {
+ long stageArrival, long stageDeadline, long period, String user,
+ ReservationId oldId) throws PlanningException {
// Initialize
ResourceCalculator resCalc = plan.getResourceCalculator();
Resource capacity = plan.getTotalCapacity();
RLESparseResourceAllocation netRLERes = plan.getAvailableResourceOverTime(
- user, oldId, stageArrival, stageDeadline, 0);
+ user, oldId, stageArrival, stageDeadline, period);
long step = plan.getStep();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java
index 294564a..b9ce54e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/BaseSharingPolicyTest.java
@@ -180,8 +180,12 @@ public abstract class BaseSharingPolicyTest {
}
}
-
- rle.addInterval(new ReservationInterval(rStart, rEnd), alloc);
+ if(rStart > rEnd){
+ rle.addInterval(new ReservationInterval(rStart, period), alloc);
+ rle.addInterval(new ReservationInterval(0, rEnd), alloc);
+ } else {
+ rle.addInterval(new ReservationInterval(rStart, rEnd), alloc);
+ }
return rle;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
index d054d3a..4ad4c3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacityOverTimePolicy.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Collection;
import net.jcip.annotations.NotThreadSafe;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
import org.junit.Test;
@@ -39,63 +40,67 @@ public class TestCapacityOverTimePolicy extends BaseSharingPolicyTest {
final static long ONEDAY = 86400 * 1000;
final static long ONEHOUR = 3600 * 1000;
final static long ONEMINUTE = 60 * 1000;
- final static String TWODAYPERIOD = "7200000";
+ final static String TWOHOURPERIOD = "7200000";
final static String ONEDAYPERIOD = "86400000";
@Parameterized.Parameters(name = "Duration {0}, height {1}," +
- " submission {2}, periodic {3})")
+ " numSubmission {2}, periodic {3})")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
// easy fit
{ONEHOUR, 0.25, 1, null, null },
- {ONEHOUR, 0.25, 1, TWODAYPERIOD, null },
+ {ONEHOUR, 0.25, 1, TWOHOURPERIOD, null },
{ONEHOUR, 0.25, 1, ONEDAYPERIOD, null },
// instantaneous high, but fit integral and inst limits
{ONEMINUTE, 0.74, 1, null, null },
- {ONEMINUTE, 0.74, 1, TWODAYPERIOD, null },
+ {ONEMINUTE, 0.74, 1, TWOHOURPERIOD, null },
{ONEMINUTE, 0.74, 1, ONEDAYPERIOD, null },
// barely fit
{ONEHOUR, 0.76, 1, null, PlanningQuotaException.class },
- {ONEHOUR, 0.76, 1, TWODAYPERIOD, PlanningQuotaException.class },
+ {ONEHOUR, 0.76, 1, TWOHOURPERIOD, PlanningQuotaException.class },
{ONEHOUR, 0.76, 1, ONEDAYPERIOD, PlanningQuotaException.class },
// overcommit with single reservation
{ONEHOUR, 1.1, 1, null, PlanningQuotaException.class },
- {ONEHOUR, 1.1, 1, TWODAYPERIOD, PlanningQuotaException.class },
+ {ONEHOUR, 1.1, 1, TWOHOURPERIOD, PlanningQuotaException.class },
{ONEHOUR, 1.1, 1, ONEDAYPERIOD, PlanningQuotaException.class },
// barely fit with multiple reservations (instantaneously, lowering to
// 1min to fit integral)
{ONEMINUTE, 0.25, 3, null, null },
- {ONEMINUTE, 0.25, 3, TWODAYPERIOD, null },
+ {ONEMINUTE, 0.25, 3, TWOHOURPERIOD, null },
{ONEMINUTE, 0.25, 3, ONEDAYPERIOD, null },
// overcommit with multiple reservations (instantaneously)
{ONEMINUTE, 0.25, 4, null, PlanningQuotaException.class },
- {ONEMINUTE, 0.25, 4, TWODAYPERIOD, PlanningQuotaException.class },
+ {ONEMINUTE, 0.25, 4, TWOHOURPERIOD, PlanningQuotaException.class },
{ONEMINUTE, 0.25, 4, ONEDAYPERIOD, PlanningQuotaException.class },
// (non-periodic) reservation longer than window
{25 * ONEHOUR, 0.25, 1, null, PlanningQuotaException.class },
- {25 * ONEHOUR, 0.25, 1, TWODAYPERIOD, PlanningQuotaException.class },
+ // NOTE: we generally don't accept periodicity < duration but the test
+ // generator will "wrap" this correctly
+ {25 * ONEHOUR, 0.25, 1, TWOHOURPERIOD, PlanningQuotaException.class },
{25 * ONEHOUR, 0.25, 1, ONEDAYPERIOD, PlanningQuotaException.class },
// (non-periodic) reservation longer than window
{25 * ONEHOUR, 0.05, 5, null, PlanningQuotaException.class },
- {25 * ONEHOUR, 0.05, 5, TWODAYPERIOD, PlanningQuotaException.class },
+ // NOTE: we generally don't accept periodicity < duration but the test
+ // generator will "wrap" this correctly
+ {25 * ONEHOUR, 0.05, 5, TWOHOURPERIOD, PlanningQuotaException.class },
{25 * ONEHOUR, 0.05, 5, ONEDAYPERIOD, PlanningQuotaException.class },
// overcommit integral
{ONEDAY, 0.26, 1, null, PlanningQuotaException.class },
- {2 * ONEHOUR, 0.26, 1, TWODAYPERIOD, PlanningQuotaException.class },
+ {2 * ONEHOUR, 0.26, 1, TWOHOURPERIOD, PlanningQuotaException.class },
{2 * ONEDAY, 0.26, 1, ONEDAYPERIOD, PlanningQuotaException.class },
// overcommit integral
{ONEDAY / 2, 0.51, 1, null, PlanningQuotaException.class },
- {2 * ONEHOUR / 2, 0.51, 1, TWODAYPERIOD,
+ {2 * ONEHOUR / 2, 0.51, 1, TWOHOURPERIOD,
PlanningQuotaException.class },
{2 * ONEDAY / 2, 0.51, 1, ONEDAYPERIOD, PlanningQuotaException.class }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
index 03569d4..7f2d199 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java
@@ -17,6 +17,7 @@
*******************************************************************************/
package org.apache.hadoop.yarn.server.resourcemanager.reservation;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -119,6 +120,18 @@ public class TestInMemoryPlan {
checkAllocation(plan, alloc, start, 0);
}
+ @Test(expected = PlanningException.class)
+ public void testOutOfRange() throws PlanningException {
+ maxPeriodicity = 100;
+ Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L,
+ resCalc, minAlloc, maxAlloc, planName, replanner, true, maxPeriodicity,
+ context, new UTCClock());
+
+ // we expect the plan to complaint as the range 330-150 > 50
+ RLESparseResourceAllocation availableBefore =
+ plan.getAvailableResourceOverTime(user, null, 150, 330, 50);
+ }
+
@Test
public void testAddPeriodicReservation() throws PlanningException {
@@ -146,8 +159,14 @@ public class TestInMemoryPlan {
checkAllocation(plan, alloc, start, period);
RLESparseResourceAllocation available =
- plan.getAvailableResourceOverTime(user, reservationID, 150, 330, 50);
- System.out.println(available);
+ plan.getAvailableResourceOverTime(user, null, 130, 170, 50);
+
+ // the reservation has period 20 starting at 10, and the interaction with
+ // the period 50 request means that every 10 we expect a "90GB" point
+ assertEquals(92160, available.getCapacityAtTime(130).getMemorySize());
+ assertEquals(92160, available.getCapacityAtTime(140).getMemorySize());
+ assertEquals(92160, available.getCapacityAtTime(150).getMemorySize());
+
}
private void checkAllocation(Plan plan, int[] alloc, int start,
@@ -162,18 +181,18 @@ public class TestInMemoryPlan {
for (int i = 0; i < alloc.length; i++) {
// only one instance for non-periodic reservation
if (periodicity <= 0) {
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
userCons.getCapacityAtTime(start + i));
} else {
// periodic reservations should repeat
long y = 0;
Resource res = Resource.newInstance(1024 * (alloc[i]), (alloc[i]));
while (y <= end * 2) {
- Assert.assertEquals("At time: " + start + i + y, res,
+ assertEquals("At time: " + start + i + y, res,
plan.getTotalCommittedResources(start + i + y));
- Assert.assertEquals(" At time: " + (start + i + y), res,
+ assertEquals(" At time: " + (start + i + y), res,
userCons.getCapacityAtTime(start + i + y));
y = y + periodicity;
}
@@ -253,9 +272,9 @@ public class TestInMemoryPlan {
RLESparseResourceAllocation userCons =
plan.getConsumptionForUserOverTime(user, start, start + alloc.length);
for (int i = 0; i < alloc.length; i++) {
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
+ assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])),
userCons.getCapacityAtTime(start + i));
}
@@ -275,9 +294,9 @@ public class TestInMemoryPlan {
start + updatedAlloc.length);
for (int i = 0; i < updatedAlloc.length; i++) {
- Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
+ assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
updatedAlloc[i] + i), plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
+ assertEquals(Resource.newInstance(1024 * (updatedAlloc[i] + i),
updatedAlloc[i] + i), userCons.getCapacityAtTime(start + i));
}
}
@@ -371,10 +390,10 @@ public class TestInMemoryPlan {
plan.getConsumptionForUserOverTime(user, start, start + alloc.length);
for (int i = 0; i < alloc.length; i++) {
- Assert.assertEquals(
+ assertEquals(
Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(
+ assertEquals(
Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)),
userCons.getCapacityAtTime(start + i));
}
@@ -389,9 +408,9 @@ public class TestInMemoryPlan {
userCons =
plan.getConsumptionForUserOverTime(user, start, start + alloc.length);
for (int i = 0; i < alloc.length; i++) {
- Assert.assertEquals(Resource.newInstance(0, 0),
+ assertEquals(Resource.newInstance(0, 0),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(0, 0),
+ assertEquals(Resource.newInstance(0, 0),
userCons.getCapacityAtTime(start + i));
}
}
@@ -492,11 +511,11 @@ public class TestInMemoryPlan {
plan.getConsumptionForUserOverTime(user, start, start + alloc2.length);
for (int i = 0; i < alloc2.length; i++) {
- Assert.assertEquals(
+ assertEquals(
Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
alloc1[i] + alloc2[i] + i),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(
+ assertEquals(
Resource.newInstance(1024 * (alloc1[i] + alloc2[i] + i),
alloc1[i] + alloc2[i] + i),
userCons.getCapacityAtTime(start + i));
@@ -530,9 +549,9 @@ public class TestInMemoryPlan {
Assert.assertNull(plan.getReservationById(reservationID1));
for (int i = 0; i < alloc1.length; i++) {
- Assert.assertEquals(Resource.newInstance(0, 0),
+ assertEquals(Resource.newInstance(0, 0),
plan.getTotalCommittedResources(start + i));
- Assert.assertEquals(Resource.newInstance(0, 0),
+ assertEquals(Resource.newInstance(0, 0),
userCons.getCapacityAtTime(start + i));
}
}
@@ -721,16 +740,16 @@ public class TestInMemoryPlan {
private void doAssertions(Plan plan, ReservationAllocation rAllocation) {
ReservationId reservationID = rAllocation.getReservationId();
Assert.assertNotNull(plan.getReservationById(reservationID));
- Assert.assertEquals(rAllocation, plan.getReservationById(reservationID));
+ assertEquals(rAllocation, plan.getReservationById(reservationID));
Assert.assertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
if (rAllocation.getPeriodicity() <= 0) {
- Assert.assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
+ assertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
}
- Assert.assertEquals(totalCapacity, plan.getTotalCapacity());
- Assert.assertEquals(minAlloc, plan.getMinimumAllocation());
- Assert.assertEquals(maxAlloc, plan.getMaximumAllocation());
- Assert.assertEquals(resCalc, plan.getResourceCalculator());
- Assert.assertEquals(planName, plan.getQueueName());
+ assertEquals(totalCapacity, plan.getTotalCapacity());
+ assertEquals(minAlloc, plan.getMinimumAllocation());
+ assertEquals(maxAlloc, plan.getMaximumAllocation());
+ assertEquals(resCalc, plan.getResourceCalculator());
+ assertEquals(planName, plan.getQueueName());
Assert.assertTrue(plan.getMoveOnExpiry());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
index 25ec9c9..9fa8559 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
@@ -26,11 +26,14 @@ import static org.mockito.Mockito.mock;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ReservationRequest;
@@ -54,11 +57,27 @@ import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+/**
+ * This class tests the {@code AlignedPlannerWithGreedy} agent.
+ */
+@RunWith(value = Parameterized.class)
+@NotThreadSafe
+@SuppressWarnings("VisibilityModifier")
public class TestAlignedPlanner {
+ @Parameterized.Parameter(value = 0)
+ public String recurrenceExpression;
+
+ final static String NONPERIODIC = "0";
+ final static String THREEHOURPERIOD = "10800000";
+ final static String ONEDAYPERIOD = "86400000";
+
private static final Logger LOG = LoggerFactory
.getLogger(TestAlignedPlanner.class);
@@ -72,6 +91,16 @@ public class TestAlignedPlanner {
private Resource clusterCapacity;
private long step;
+
+ @Parameterized.Parameters(name = "Testing: periodicity {0})")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {NONPERIODIC},
+ {THREEHOURPERIOD},
+ {ONEDAYPERIOD}
+ });
+ }
+
@Test
public void testSingleReservationAccept() throws PlanningException {
@@ -107,6 +136,9 @@ public class TestAlignedPlanner {
assertTrue(alloc1.toString(),
check(alloc1, 10 * step, 20 * step, 10, 2048, 2));
+ System.out.println("--------AFTER AGENT----------");
+ System.out.println(plan.toString());
+
}
@Test
@@ -1139,9 +1171,11 @@ public class TestAlignedPlanner {
long deadline, ReservationRequest[] reservationRequests,
ReservationRequestInterpreter rType, String username) {
- return ReservationDefinition.newInstance(arrival, deadline,
- ReservationRequests.newInstance(Arrays.asList(reservationRequests),
- rType), username);
+
+ return ReservationDefinition.newInstance(arrival,
+ deadline, ReservationRequests
+ .newInstance(Arrays.asList(reservationRequests), rType),
+ username, recurrenceExpression, Priority.UNDEFINED);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
index 51b971b..7c7b0a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -59,13 +59,20 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RunWith(Parameterized.class)
+@SuppressWarnings("VisibilityModifier")
public class TestGreedyReservationAgent {
+
+ @Parameterized.Parameter(value = 0)
+ public boolean allocateLeft;
+
+ @Parameterized.Parameter(value = 1)
+ public String recurrenceExpression;
+
private static final Logger LOG = LoggerFactory
.getLogger(TestGreedyReservationAgent.class);
@@ -76,16 +83,18 @@ public class TestGreedyReservationAgent {
Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
Random rand = new Random();
long step;
- boolean allocateLeft;
-
- public TestGreedyReservationAgent(Boolean b){
- this.allocateLeft = b;
- }
- @Parameters
+ @Parameterized.Parameters(name = "Testing: allocateLeft {0}," +
+ " recurrenceExpression {1})")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
- {true}, {false}});
+ {true, "0"},
+ {false, "0"},
+ {true, "7200000"},
+ {false, "7200000"},
+ {true, "86400000"},
+ {false, "86400000"}
+ });
}
@Before
@@ -134,6 +143,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(5 * step);
rr.setDeadline(20 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequest r = ReservationRequest.newInstance(
Resource.newInstance(2048, 2), 10, 5, 10 * step);
ReservationRequests reqs = new ReservationRequestsPBImpl();
@@ -193,6 +203,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(5 * step);
rr.setDeadline(100 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequest r =
ReservationRequest.newInstance(Resource.newInstance(2048, 2), 20, 20,
10 * step);
@@ -283,8 +294,9 @@ public class TestGreedyReservationAgent {
int[] f = { 100, 100 };
ReservationDefinition rDef =
- ReservationSystemTestUtil.createSimpleReservationDefinition(
- 30 * step, 30 * step + f.length * step, f.length * step);
+ ReservationSystemTestUtil.createSimpleReservationDefinition(30 * step,
+ 30 * step + f.length * step, f.length * step, 1,
+ recurrenceExpression);
assertTrue(plan.toString(),
plan.addReservation(new InMemoryReservationAllocation(
ReservationSystemTestUtil.getNewReservationId(), rDef, "u1",
@@ -296,6 +308,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(0 * step);
rr.setDeadline(70 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER);
ReservationRequest r = ReservationRequest.newInstance(
@@ -346,10 +359,11 @@ public class TestGreedyReservationAgent {
prepareBasicPlan();
// create a completely utilized segment at time 30
int[] f = { 100, 100 };
- ReservationDefinition rDef =
- ReservationSystemTestUtil.createSimpleReservationDefinition(
- 30, 30 * step + f.length * step, f.length * step);
- assertTrue(plan.toString(),
+ ReservationDefinition rDef = ReservationSystemTestUtil
+ .createSimpleReservationDefinition(30, 30 * step + f.length * step,
+ f.length * step, 1, recurrenceExpression);
+ assertTrue(
+ plan.toString(),
plan.addReservation(new InMemoryReservationAllocation(
ReservationSystemTestUtil.getNewReservationId(), rDef, "u1",
"dedicated", 30 * step, 30 * step + f.length * step,
@@ -406,6 +420,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(0 * step);
rr.setDeadline(60 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP);
ReservationRequest r = ReservationRequest.newInstance(
@@ -453,6 +468,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(100 * step);
rr.setDeadline(120 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
ReservationRequest r = ReservationRequest.newInstance(
@@ -494,6 +510,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(100 * step);
rr.setDeadline(120 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ANY);
ReservationRequest r = ReservationRequest.newInstance(
@@ -542,6 +559,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(100L);
rr.setDeadline(120L);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ANY);
@@ -587,6 +605,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(100 * step);
rr.setDeadline(120 * step);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
ReservationRequest r = ReservationRequest.newInstance(
@@ -635,6 +654,7 @@ public class TestGreedyReservationAgent {
ReservationDefinition rr = new ReservationDefinitionPBImpl();
rr.setArrival(100L);
rr.setDeadline(120L);
+ rr.setRecurrenceExpression(recurrenceExpression);
ReservationRequests reqs = new ReservationRequestsPBImpl();
reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
ReservationRequest r = ReservationRequest.newInstance(
@@ -759,25 +779,4 @@ public class TestGreedyReservationAgent {
+ " in " + (end - start) + "ms");
}
- public static void main(String[] arg) {
-
- boolean left = false;
- // run a stress test with by default 1000 random jobs
- int numJobs = 1000;
- if (arg.length > 0) {
- numJobs = Integer.parseInt(arg[0]);
- }
- if (arg.length > 1) {
- left = Boolean.parseBoolean(arg[1]);
- }
-
- try {
- TestGreedyReservationAgent test = new TestGreedyReservationAgent(left);
- test.setup();
- test.testStress(numJobs);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6e614e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java
new file mode 100644
index 0000000..386fa68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestReservationAgents.java
@@ -0,0 +1,213 @@
+/*****************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *****************************************************************************/
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityOverTimePolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Random;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * General purpose ReservationAgent tester.
+ */
+@RunWith(Parameterized.class)
+@SuppressWarnings("VisibilityModifier")
+public class TestReservationAgents {
+
+ @Parameterized.Parameter(value = 0)
+ public Class agentClass;
+
+ @Parameterized.Parameter(value = 1)
+ public boolean allocateLeft;
+
+ @Parameterized.Parameter(value = 2)
+ public String recurrenceExpression;
+
+ @Parameterized.Parameter(value = 3)
+ public int numOfNodes;
+
+ private long step;
+ private Random rand = new Random(2);
+ private ReservationAgent agent;
+ private Plan plan;
+ private ResourceCalculator resCalc = new DefaultResourceCalculator();
+ private Resource minAlloc = Resource.newInstance(1024, 1);
+ private Resource maxAlloc = Resource.newInstance(32 * 1023, 32);
+
+ private long timeHorizon = 2 * 24 * 3600 * 1000; // 2 days
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestReservationAgents.class);
+
+ @Parameterized.Parameters(name = "Testing: agent {0}, allocateLeft: {1}," +
+ " recurrenceExpression: {2}, numNodes: {3})")
+ public static Collection<Object[]> data() {
+ return Arrays.asList(
+ new Object[][] {{GreedyReservationAgent.class, true, "0", 100 },
+ {GreedyReservationAgent.class, false, "0", 100 },
+ {GreedyReservationAgent.class, true, "7200000", 100 },
+ {GreedyReservationAgent.class, false, "7200000", 100 },
+ {GreedyReservationAgent.class, true, "86400000", 100 },
+ {GreedyReservationAgent.class, false, "86400000", 100 },
+ {AlignedPlannerWithGreedy.class, true, "0", 100 },
+ {AlignedPlannerWithGreedy.class, false, "0", 100 },
+ {AlignedPlannerWithGreedy.class, true, "7200000", 100 },
+ {AlignedPlannerWithGreedy.class, false, "7200000", 100 },
+ {AlignedPlannerWithGreedy.class, true, "86400000", 100 },
+ {AlignedPlannerWithGreedy.class, false, "86400000", 100 } });
+ }
+
+ @Before
+ public void setup() throws Exception {
+
+ long seed = rand.nextLong();
+ rand.setSeed(seed);
+ LOG.info("Running with seed: " + seed);
+
+ // setting completely loose quotas
+ long timeWindow = 1000000L;
+ Resource clusterCapacity =
+ Resource.newInstance(numOfNodes * 1024, numOfNodes);
+ step = 1000L;
+ String reservationQ =
+ ReservationSystemTestUtil.getFullReservationQueueName();
+
+ float instConstraint = 100;
+ float avgConstraint = 100;
+
+ ReservationSchedulerConfiguration conf = ReservationSystemTestUtil
+ .createConf(reservationQ, timeWindow, instConstraint, avgConstraint);
+ CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
+ policy.init(reservationQ, conf);
+
+ // setting conf to
+ conf.setBoolean(GreedyReservationAgent.FAVOR_EARLY_ALLOCATION,
+ allocateLeft);
+ agent = (ReservationAgent) agentClass.newInstance();
+ agent.init(conf);
+
+ QueueMetrics queueMetrics = mock(QueueMetrics.class);
+ RMContext context = ReservationSystemTestUtil.createMockRMContext();
+
+ plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
+ resCalc, minAlloc, maxAlloc, "dedicated", null, true, context);
+ }
+
+ @Test
+ public void test() throws Exception {
+
+ long period = Long.parseLong(recurrenceExpression);
+ for (int i = 0; i < 1000; i++) {
+ ReservationDefinition rr = createRandomRequest(i);
+ if (rr != null) {
+ ReservationId reservationID =
+ ReservationSystemTestUtil.getNewReservationId();
+ try {
+ agent.createReservation(reservationID, "u1", plan, rr);
+ } catch (PlanningException p) {
+ // happens
+ }
+ }
+ }
+
+ }
+
+ private ReservationDefinition createRandomRequest(int i)
+ throws PlanningException {
+ long arrival = (long) Math.floor(rand.nextDouble() * timeHorizon);
+ long period = Long.parseLong(recurrenceExpression);
+
+ // min between period and rand around 30min
+ long duration =
+ (long) Math.round(Math.min(rand.nextDouble() * 3600 * 1000, period));
+
+ // min between period and rand around 5x duration
+ long deadline = (long) Math
+ .ceil(arrival + Math.min(duration * rand.nextDouble() * 10, period));
+
+ assert((deadline - arrival) <= period);
+
+ RLESparseResourceAllocation available = plan
+ .getAvailableResourceOverTime("u1", null, arrival, deadline, period);
+ NavigableMap<Long, Resource> availableMap = available.getCumulative();
+
+ // look at available space, and for each segment, use half of it with 50%
+ // probability
+ List<ReservationRequest> reservationRequests = new ArrayList<>();
+ for (Map.Entry<Long, Resource> e : availableMap.entrySet()) {
+ if (e.getValue() != null && rand.nextDouble() > 0.001) {
+ int numContainers = (int) Math.ceil(Resources.divide(resCalc,
+ plan.getTotalCapacity(), e.getValue(), minAlloc) / 2);
+ long tempDur =
+ Math.min(duration, availableMap.higherKey(e.getKey()) - e.getKey());
+ reservationRequests.add(ReservationRequest.newInstance(minAlloc,
+ numContainers, 1, tempDur));
+ }
+ }
+
+ if (reservationRequests.size() < 1) {
+ return null;
+ }
+
+ ReservationDefinition rr = new ReservationDefinitionPBImpl();
+ rr.setArrival(arrival);
+ rr.setDeadline(deadline);
+ rr.setRecurrenceExpression(recurrenceExpression);
+ ReservationRequests reqs = new ReservationRequestsPBImpl();
+ reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER);
+ reqs.setReservationResources(reservationRequests);
+ rr.setReservationRequests(reqs);
+ rr.setReservationName("res_" + i);
+
+ return rr;
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-14459. SerializationFactory
shouldn't throw a NullPointerException if the serializations list is not
defined (Contributed by Nandor Kollar via Daniel Templeton)
Posted by jh...@apache.org.
HADOOP-14459. SerializationFactory shouldn't throw a NullPointerException if the serializations list is not defined
(Contributed by Nandor Kollar via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20e9ce3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20e9ce3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20e9ce3a
Branch: refs/heads/YARN-5734
Commit: 20e9ce3ab33599d2ac6859a8319e1ce0bf0a4363
Parents: acf5b88
Author: Daniel Templeton <te...@apache.org>
Authored: Wed Oct 4 15:04:02 2017 +0200
Committer: Daniel Templeton <te...@apache.org>
Committed: Wed Oct 4 15:04:02 2017 +0200
----------------------------------------------------------------------
.../io/serializer/SerializationFactory.java | 18 ++++++------------
.../io/serializer/TestSerializationFactory.java | 13 +++++++++++++
2 files changed, 19 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e9ce3a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
index 969ca3a..ce0c3fe 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
@@ -55,18 +55,12 @@ public class SerializationFactory extends Configured {
*/
public SerializationFactory(Configuration conf) {
super(conf);
- if (conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY).equals("")) {
- LOG.warn("Serialization for various data types may not be available. Please configure "
- + CommonConfigurationKeys.IO_SERIALIZATIONS_KEY
- + " properly to have serialization support (it is currently not set).");
- } else {
- for (String serializerName : conf.getTrimmedStrings(
- CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, new String[] {
- WritableSerialization.class.getName(),
- AvroSpecificSerialization.class.getName(),
- AvroReflectSerialization.class.getName() })) {
- add(conf, serializerName);
- }
+ for (String serializerName : conf.getTrimmedStrings(
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY,
+ new String[]{WritableSerialization.class.getName(),
+ AvroSpecificSerialization.class.getName(),
+ AvroReflectSerialization.class.getName()})) {
+ add(conf, serializerName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e9ce3a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
index 2cde3e3..d9a0009 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
@@ -51,6 +51,19 @@ public class TestSerializationFactory {
SerializationFactory factory = new SerializationFactory(conf);
}
+ /**
+ * Test the case when {@code IO_SERIALIZATIONS_KEY}
+ * is not set at all, because something unset this key.
+ * This shouldn't result in any error, the defaults present
+ * in construction should be used in this case.
+ */
+ @Test
+ public void testSerializationKeyIsUnset() {
+ Configuration conf = new Configuration();
+ conf.unset(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY);
+ SerializationFactory factory = new SerializationFactory(conf);
+ }
+
@Test
public void testSerializationKeyIsInvalid() {
Configuration conf = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-2162. Add ability in Fair
Scheduler to optionally configure maxResources in terms of
percentage(addendum). (Yufei Gu)
Posted by jh...@apache.org.
YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage(addendum). (Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49ae5381
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49ae5381
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49ae5381
Branch: refs/heads/YARN-5734
Commit: 49ae538164bf631ebf961006a06c8ba0f3469b89
Parents: 99292ad
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 6 00:08:25 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 6 00:08:25 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/ConfigurableResource.java | 72 ++++++++++++++++++++
.../fair/TestConfigurableResource.java | 68 ++++++++++++++++++
2 files changed, 140 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ae5381/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
new file mode 100644
index 0000000..8de41be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * A {@code ConfigurableResource} object represents an entity that is used to
+ * configure resources, such as maximum resources of a queue. It can be
+ * percentage of cluster resources or an absolute value.
+ */
+@Private
+@Unstable
+public class ConfigurableResource {
+ private final Resource resource;
+ private final double[] percentages;
+
+ public ConfigurableResource(double[] percentages) {
+ this.percentages = percentages.clone();
+ this.resource = null;
+ }
+
+ public ConfigurableResource(Resource resource) {
+ this.percentages = null;
+ this.resource = resource;
+ }
+
+ /**
+ * Get resource by multiplying the cluster resource and the percentage of
+ * each resource respectively. Return the absolute resource if either
+ * {@code percentages} or {@code clusterResource) is null.
+ *
+ * @param clusterResource the cluster resource
+ * @return resource
+ */
+ public Resource getResource(Resource clusterResource) {
+ if (percentages != null && clusterResource != null) {
+ long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
+ int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
+ return Resource.newInstance(memory, vcore);
+ } else {
+ return resource;
+ }
+ }
+
+ /**
+ * Get the absolute resource.
+ *
+ * @return the absolute resource
+ */
+ public Resource getResource() {
+ return resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ae5381/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
new file mode 100644
index 0000000..249d1f7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestConfigurableResource.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * To test class {@link ConfigurableResource}.
+ */
+public class TestConfigurableResource {
+ private final Resource clusterResource = Resources.createResource(2048, 2);
+
+ @Test
+ public void testGetResourceWithPercentage() {
+ ConfigurableResource configurableResource =
+ new ConfigurableResource(new double[] {0.5, 0.5});
+ assertEquals(
+ configurableResource.getResource(clusterResource).getMemorySize(),
+ 1024);
+ assertEquals(
+ configurableResource.getResource(clusterResource).getVirtualCores(), 1);
+
+ assertNull("The absolute resource should be null since object"
+ + " configurableResource is initialized with percentages",
+ configurableResource.getResource());
+ assertNull("The absolute resource should be null since cluster resource"
+ + " is null", configurableResource.getResource(null));
+ }
+
+ @Test
+ public void testGetResourceWithAbsolute() {
+ ConfigurableResource configurableResource =
+ new ConfigurableResource(Resources.createResource(3072, 3));
+ assertEquals(configurableResource.getResource().getMemorySize(), 3072);
+ assertEquals(configurableResource.getResource().getVirtualCores(), 3);
+
+ assertEquals(
+ configurableResource.getResource(clusterResource).getMemorySize(),
+ 3072);
+ assertEquals(
+ configurableResource.getResource(clusterResource).getVirtualCores(),
+ 3);
+
+ assertEquals(configurableResource.getResource(null).getMemorySize(), 3072);
+ assertEquals(configurableResource.getResource(null).getVirtualCores(), 3);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-7252. Removing queue then
failing over results in exception
Posted by jh...@apache.org.
YARN-7252. Removing queue then failing over results in exception
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8c68695
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8c68695
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8c68695
Branch: refs/heads/YARN-5734
Commit: b8c68695a7e343920acd0df9128628ba178aa564
Parents: 3a5d3f8
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Tue Sep 26 11:41:05 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:54 2017 -0700
----------------------------------------------------------------------
.../capacity/CapacitySchedulerContext.java | 6 ++
.../capacity/CapacitySchedulerQueueManager.java | 11 +-
.../capacity/conf/TestZKConfigurationStore.java | 102 +++++++++++++++++++
3 files changed, 117 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8c68695/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
index 9aeaec6..7c918a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
@@ -88,4 +88,10 @@ public interface CapacitySchedulerContext {
* @return Max Cluster level App priority.
*/
Priority getMaxClusterLevelAppPriority();
+
+ /**
+ * Returns if configuration is mutable.
+ * @return if configuration is mutable
+ */
+ boolean isConfigurationMutable();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8c68695/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 1ceb6fb..48c289f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueState;
@@ -170,8 +171,14 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
CSQueue newRoot = parseQueue(this.csContext, newConf, null,
CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP);
- // Ensure queue hiearchy in the new XML file is proper.
- validateQueueHierarchy(queues, newQueues);
+ // When failing over, if using configuration store, don't validate queue
+ // hierarchy since queues can be removed without being STOPPED.
+ if (!csContext.isConfigurationMutable() ||
+ csContext.getRMContext().getHAServiceState()
+ != HAServiceProtocol.HAServiceState.STANDBY) {
+ // Ensure queue hiearchy in the new XML file is proper.
+ validateQueueHierarchy(queues, newQueues);
+ }
// Add new queues and delete OldQeueus only after validation.
updateQueues(queues, newQueues);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8c68695/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 355f741..0cf5e6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -38,18 +38,21 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfSchedu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
/**
* Tests {@link ZKConfigurationStore}.
@@ -303,6 +306,105 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
rm2.close();
}
+ /**
+ * When failing over, if RM1 stopped and removed a queue that RM2 has in
+ * memory, failing over to RM2 should not throw an exception.
+ * @throws Exception
+ */
+ @Test
+ public void testFailoverAfterRemoveQueue() throws Exception {
+ HAServiceProtocol.StateChangeRequestInfo req =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+
+ Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234);
+ ResourceManager rm1 = new MockRM(conf1);
+ rm1.start();
+ rm1.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm1.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678);
+ ResourceManager rm2 = new MockRM(conf2);
+ rm2.start();
+ assertEquals("RM should be Standby",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ // Add root.a
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ Map<String, String> addParams = new HashMap<>();
+ addParams.put("capacity", "100");
+ QueueConfigInfo addInfo = new QueueConfigInfo("root.a", addParams);
+ schedConfUpdateInfo.getAddQueueInfo().add(addInfo);
+ // Stop root.default
+ Map<String, String> stopParams = new HashMap<>();
+ stopParams.put("state", "STOPPED");
+ stopParams.put("capacity", "0");
+ QueueConfigInfo stopInfo = new QueueConfigInfo("root.default", stopParams);
+ schedConfUpdateInfo.getUpdateQueueInfo().add(stopInfo);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ confProvider.confirmPendingMutation(true);
+ assertTrue(Arrays.asList(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues").split
+ (",")).contains("a"));
+
+ // Remove root.default
+ schedConfUpdateInfo.getUpdateQueueInfo().clear();
+ schedConfUpdateInfo.getAddQueueInfo().clear();
+ schedConfUpdateInfo.getRemoveQueueInfo().add("root.default");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ confProvider.confirmPendingMutation(true);
+ assertEquals("a", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues"));
+
+ // Start RM2 and verifies it starts with updated configuration
+ rm2.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm2.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
+ if (HAServiceProtocol.HAServiceState.ACTIVE ==
+ rm1.getRMContext().getRMAdminService().getServiceStatus()
+ .getState()) {
+ Thread.sleep(100);
+ }
+ }
+ assertEquals("RM should have been fenced",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ assertEquals("a", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve()
+ .get("yarn.scheduler.capacity.root.queues"));
+ assertEquals("a", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("yarn.scheduler.capacity.root.queues"));
+ // Transition to standby will set RM's HA status and then reinitialize in
+ // a separate thread. Despite asserting for STANDBY state, it's
+ // possible for reinitialization to be unfinished. Wait here for it to
+ // finish, otherwise closing rm1 will close zkManager and the unfinished
+ // reinitialization will throw an exception.
+ Thread.sleep(10000);
+ rm1.close();
+ rm2.close();
+ }
+
@Override
public YarnConfigurationStore createConfStore() {
return new ZKConfigurationStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-6975. Moving logging APIs
over to slf4j in hadoop-yarn-server-tests,
hadoop-yarn-server-web-proxy and hadoop-yarn-server-router. Contributed by
Yeliang Cang.
Posted by jh...@apache.org.
YARN-6975. Moving logging APIs over to slf4j in hadoop-yarn-server-tests, hadoop-yarn-server-web-proxy and hadoop-yarn-server-router. Contributed by Yeliang Cang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acf5b880
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acf5b880
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acf5b880
Branch: refs/heads/YARN-5734
Commit: acf5b880d8283f5a96455c89a6b1b548d0d8c0e7
Parents: b34b3ff
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 4 14:53:41 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 4 14:53:41 2017 +0900
----------------------------------------------------------------------
.../yarn/server/router/webapp/RouterWebServiceUtil.java | 8 ++++----
.../hadoop/yarn/server/ContainerTokenIdentifierForTest.java | 6 +++---
.../java/org/apache/hadoop/yarn/server/MiniYARNCluster.java | 7 ++++---
.../hadoop/yarn/server/NMTokenIdentifierNewForTest.java | 6 +++---
.../hadoop/yarn/server/TestContainerManagerSecurity.java | 6 +++---
.../java/org/apache/hadoop/yarn/server/TestDiskFailures.java | 6 +++---
.../apache/hadoop/yarn/server/webproxy/AppReportFetcher.java | 3 ---
7 files changed, 20 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index e769a86..5528e78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -33,8 +33,6 @@ import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
@@ -54,6 +52,8 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
import com.sun.jersey.core.util.MultivaluedMapImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The Router webservice util class.
@@ -62,8 +62,8 @@ public final class RouterWebServiceUtil {
private static String user = "YarnRouter";
- private static final Log LOG =
- LogFactory.getLog(RouterWebServiceUtil.class.getName());
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RouterWebServiceUtil.class.getName());
private final static String PARTIAL_REPORT = "Partial Report ";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
index 99b3be5..0d4d2e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/ContainerTokenIdentifierForTest.java
@@ -23,8 +23,6 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
@@ -40,10 +38,12 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
import org.apache.hadoop.yarn.proto.YarnSecurityTestTokenProtos.ContainerTokenIdentifierForTestProto;
import com.google.protobuf.TextFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ContainerTokenIdentifierForTest extends ContainerTokenIdentifier {
- private static Log LOG = LogFactory.getLog(ContainerTokenIdentifier.class);
+ private static Logger LOG = LoggerFactory.getLogger(ContainerTokenIdentifier.class);
public static final Text KIND = new Text("ContainerToken");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 27aa916..7a225c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -28,8 +28,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -100,6 +98,8 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES;
@@ -126,7 +126,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourc
@InterfaceStability.Evolving
public class MiniYARNCluster extends CompositeService {
- private static final Log LOG = LogFactory.getLog(MiniYARNCluster.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MiniYARNCluster.class);
// temp fix until metrics system can auto-detect itself running in unit test:
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
index 8153b44..7d21b48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/NMTokenIdentifierNewForTest.java
@@ -23,8 +23,6 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -36,10 +34,12 @@ import org.apache.hadoop.yarn.proto.YarnSecurityTestTokenProtos.NMTokenIdentifie
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import com.google.protobuf.TextFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class NMTokenIdentifierNewForTest extends NMTokenIdentifier {
- private static Log LOG = LogFactory.getLog(NMTokenIdentifierNewForTest.class);
+ private static Logger LOG = LoggerFactory.getLogger(NMTokenIdentifierNewForTest.class);
public static final Text KIND = new Text("NMToken");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 3ba4beb..1cbad70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -29,8 +29,6 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputBuffer;
@@ -83,12 +81,14 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@RunWith(Parameterized.class)
public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
- static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class);
+ static Logger LOG = LoggerFactory.getLogger(TestContainerManagerSecurity.class);
static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
private static MiniYARNCluster yarnCluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
index bf82ec5..e9de20a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestDiskFailures.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
@@ -45,6 +43,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Verify if NodeManager's in-memory good local dirs list and good log dirs list
@@ -54,7 +54,7 @@ import org.junit.Assert;
*/
public class TestDiskFailures {
- private static final Log LOG = LogFactory.getLog(TestDiskFailures.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestDiskFailures.class);
private static final long DISK_HEALTH_CHECK_INTERVAL = 1000;//1 sec
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acf5b880/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
index 11ec2e4..6b79e19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.webproxy;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -42,7 +40,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
*/
public class AppReportFetcher {
enum AppReportSource { RM, AHS }
- private static final Log LOG = LogFactory.getLog(AppReportFetcher.class);
private final Configuration conf;
private final ApplicationClientProtocol applicationsManager;
private final ApplicationHistoryProtocol historyManager;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-5946: Create
YarnConfigurationStore interface and InMemoryConfigurationStore class.
Contributed by Jonathan Hung
Posted by jh...@apache.org.
YARN-5946: Create YarnConfigurationStore interface and
InMemoryConfigurationStore class. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1a4709c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1a4709c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1a4709c
Branch: refs/heads/YARN-5734
Commit: c1a4709cec15aa874abde87d8cca0739be400568
Parents: ae0f61e
Author: Xuan <xg...@apache.org>
Authored: Fri Feb 24 15:58:12 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:52 2017 -0700
----------------------------------------------------------------------
.../conf/InMemoryConfigurationStore.java | 86 +++++++++++
.../capacity/conf/YarnConfigurationStore.java | 154 +++++++++++++++++++
.../conf/TestYarnConfigurationStore.java | 70 +++++++++
3 files changed, 310 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a4709c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
new file mode 100644
index 0000000..a208fb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A default implementation of {@link YarnConfigurationStore}. Doesn't offer
+ * persistent configuration storage, just stores the configuration in memory.
+ */
+public class InMemoryConfigurationStore implements YarnConfigurationStore {
+
+ private Configuration schedConf;
+ private LinkedList<LogMutation> pendingMutations;
+ private long pendingId;
+
+ @Override
+ public void initialize(Configuration conf, Configuration schedConf) {
+ this.schedConf = schedConf;
+ this.pendingMutations = new LinkedList<>();
+ this.pendingId = 0;
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation) {
+ logMutation.setId(++pendingId);
+ pendingMutations.add(logMutation);
+ return pendingId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid) {
+ LogMutation mutation = pendingMutations.poll();
+ // If confirmMutation is called out of order, discard mutations until id
+ // is reached.
+ while (mutation != null) {
+ if (mutation.getId() == id) {
+ if (isValid) {
+ Map<String, String> mutations = mutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ return true;
+ }
+ mutation = pendingMutations.poll();
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ return schedConf;
+ }
+
+ @Override
+ public synchronized List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a4709c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
new file mode 100644
index 0000000..22c0ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * YarnConfigurationStore exposes the methods needed for retrieving and
+ * persisting {@link CapacityScheduler} configuration via key-value
+ * using write-ahead logging. When configuration mutation is requested, caller
+ * should first log it with {@code logMutation}, which persists this pending
+ * mutation. This mutation is merged to the persisted configuration only after
+ * {@code confirmMutation} is called.
+ *
+ * On startup/recovery, caller should call {@code retrieve} to get all
+ * confirmed mutations, then get pending mutations which were not confirmed via
+ * {@code getPendingMutations}, and replay/confirm them via
+ * {@code confirmMutation} as in the normal case.
+ */
+public interface YarnConfigurationStore {
+
+ /**
+ * LogMutation encapsulates the fields needed for configuration mutation
+ * audit logging and recovery.
+ */
+ class LogMutation {
+ private Map<String, String> updates;
+ private String user;
+ private long id;
+
+ /**
+ * Create log mutation prior to logging.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ */
+ public LogMutation(Map<String, String> updates, String user) {
+ this(updates, user, 0);
+ }
+
+ /**
+ * Create log mutation for recovery.
+ * @param updates key-value configuration updates
+ * @param user user who requested configuration change
+ * @param id transaction id of configuration change
+ */
+ LogMutation(Map<String, String> updates, String user, long id) {
+ this.updates = updates;
+ this.user = user;
+ this.id = id;
+ }
+
+ /**
+ * Get key-value configuration updates.
+ * @return map of configuration updates
+ */
+ public Map<String, String> getUpdates() {
+ return updates;
+ }
+
+ /**
+ * Get user who requested configuration change.
+ * @return user who requested configuration change
+ */
+ public String getUser() {
+ return user;
+ }
+
+ /**
+ * Get transaction id of this configuration change.
+ * @return transaction id
+ */
+ public long getId() {
+ return id;
+ }
+
+ /**
+ * Set transaction id of this configuration change.
+ * @param id transaction id
+ */
+ public void setId(long id) {
+ this.id = id;
+ }
+ }
+
+ /**
+ * Initialize the configuration store.
+ * @param conf configuration to initialize store with
+ * @param schedConf Initial key-value configuration to persist
+ */
+ void initialize(Configuration conf, Configuration schedConf);
+
+ /**
+ * Logs the configuration change to backing store. Generates an id associated
+ * with this mutation, sets it in {@code logMutation}, and returns it.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @return id which configuration store associates with this mutation
+ */
+ long logMutation(LogMutation logMutation);
+
+ /**
+ * Should be called after {@code logMutation}. Gets the pending mutation
+ * associated with {@code id} and marks the mutation as persisted (no longer
+ * pending). If isValid is true, merge the mutation with the persisted
+ * configuration.
+ *
+ * If {@code confirmMutation} is called with ids in a different order than
+ * was returned by {@code logMutation}, the result is implementation
+ * dependent.
+ * @param id id of mutation to be confirmed
+ * @param isValid if true, update persisted configuration with mutation
+ * associated with {@code id}.
+ * @return true on success
+ */
+ boolean confirmMutation(long id, boolean isValid);
+
+ /**
+ * Retrieve the persisted configuration.
+ * @return configuration as key-value
+ */
+ Configuration retrieve();
+
+ /**
+ * Get the list of pending mutations, in the order they were logged.
+ * @return list of mutations
+ */
+ List<LogMutation> getPendingMutations();
+
+ /**
+ * Get a list of confirmed configuration mutations starting from a given id.
+ * @param fromId id from which to start getting mutations, inclusive
+ * @return list of configuration mutations
+ */
+ List<LogMutation> getConfirmedConfHistory(long fromId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1a4709c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
new file mode 100644
index 0000000..dff4e77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestYarnConfigurationStore {
+
+ private YarnConfigurationStore confStore;
+ private Configuration schedConf;
+
+ private static final String testUser = "testUser";
+
+ @Before
+ public void setUp() {
+ schedConf = new Configuration(false);
+ schedConf.set("key1", "val1");
+ }
+
+ @Test
+ public void testInMemoryConfigurationStore() {
+ confStore = new InMemoryConfigurationStore();
+ confStore.initialize(new Configuration(), schedConf);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ LogMutation mutation1 = new LogMutation(update1, testUser);
+ long id = confStore.logMutation(mutation1);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+ assertEquals(0, confStore.getPendingMutations().size());
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ LogMutation mutation2 = new LogMutation(update2, testUser);
+ id = confStore.logMutation(mutation2);
+ assertEquals(1, confStore.getPendingMutations().size());
+ confStore.confirmMutation(id, false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ assertEquals(0, confStore.getPendingMutations().size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-6575. Support global
configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Posted by jh...@apache.org.
YARN-6575. Support global configuration mutation in MutableConfProvider. (Jonathan Hung via Xuan Gong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a979ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a979ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a979ac
Branch: refs/heads/YARN-5734
Commit: d9a979ac5e12d2de38c9d693bb1f445566eb7279
Parents: 87f3fce
Author: Xuan <xg...@apache.org>
Authored: Mon Jun 5 16:30:38 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../ConfigurationMutationACLPolicy.java | 4 +-
.../DefaultConfigurationMutationACLPolicy.java | 4 +-
.../scheduler/MutableConfScheduler.java | 4 +-
.../scheduler/MutableConfigurationProvider.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../conf/MutableCSConfigurationProvider.java | 10 +-
...ueueAdminConfigurationMutationACLPolicy.java | 22 +++-
.../resourcemanager/webapp/RMWebServices.java | 4 +-
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 -----------
.../webapp/dao/SchedConfUpdateInfo.java | 69 +++++++++++++
.../TestConfigurationMutationACLPolicies.java | 28 ++++-
.../TestMutableCSConfigurationProvider.java | 10 +-
.../TestRMWebServicesConfigurationMutation.java | 101 +++++++++++++------
13 files changed, 205 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
index 724487b..3a388fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Interface for determining whether configuration mutations are allowed.
@@ -41,7 +41,7 @@ public interface ConfigurationMutationACLPolicy {
* @param confUpdate configurations to be updated
* @return whether provided mutation is allowed or not
*/
- boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ boolean isMutationAllowed(UserGroupInformation user, SchedConfUpdateInfo
confUpdate);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
index 680c3b8..6648668 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
/**
* Default configuration mutation ACL policy. Checks if user is YARN admin.
@@ -39,7 +39,7 @@ public class DefaultConfigurationMutationACLPolicy implements
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
return authorizer.isAdmin(user);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 93a935e..027d944 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -36,7 +36,7 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException;
+ SchedConfUpdateInfo confUpdate) throws IOException;
/**
* Get the scheduler configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index f04c128..6b8306c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
@@ -34,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4f89f7e..4d367e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,7 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -2617,7 +2617,7 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
user, confUpdate);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 8b879b0..eb97260 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
@@ -98,7 +98,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public void mutateConfiguration(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) throws IOException {
+ SchedConfUpdateInfo confUpdate) throws IOException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
" queues.");
@@ -126,7 +126,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ SchedConfUpdateInfo mutationInfo) throws IOException {
CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
CapacitySchedulerConfiguration proposedConf =
new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
@@ -140,6 +140,10 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
updateQueue(updateQueueInfo, proposedConf, confUpdate);
}
+ for (Map.Entry<String, String> global : mutationInfo.getGlobalParams()
+ .entrySet()) {
+ confUpdate.put(global.getKey(), global.getValue());
+ }
return confUpdate;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
index 1f94c1c..0a82d50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -22,15 +22,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
/**
@@ -40,16 +42,29 @@ import java.util.Set;
public class QueueAdminConfigurationMutationACLPolicy implements
ConfigurationMutationACLPolicy {
+ private Configuration conf;
private RMContext rmContext;
+ private YarnAuthorizationProvider authorizer;
@Override
- public void init(Configuration conf, RMContext context) {
+ public void init(Configuration config, RMContext context) {
+ this.conf = config;
this.rmContext = context;
+ this.authorizer = YarnAuthorizationProvider.getInstance(conf);
}
@Override
public boolean isMutationAllowed(UserGroupInformation user,
- QueueConfigsUpdateInfo confUpdate) {
+ SchedConfUpdateInfo confUpdate) {
+ // If there are global config changes, check if user is admin.
+ Map<String, String> globalParams = confUpdate.getGlobalParams();
+ if (globalParams != null && globalParams.size() != 0) {
+ if (!authorizer.isAdmin(user)) {
+ return false;
+ }
+ }
+
+ // Check if user is admin of all modified queues.
Set<String> queues = new HashSet<>();
for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
queues.add(addQueueInfo.getQueue());
@@ -71,7 +86,6 @@ public class QueueAdminConfigurationMutationACLPolicy implements
// Queue is not found, do nothing.
}
String parentPath = queuePath;
- // TODO: handle global config change.
while (queueInfo == null) {
// We are adding a queue (whose parent we are possibly also adding).
// Check ACL of lowest parent queue which already exists.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d95465b..b97e212 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2459,11 +2459,11 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@PUT
- @Path("/queues")
+ @Path("/sched-conf")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ public Response updateSchedulerConfiguration(SchedConfUpdateInfo
mutationInfo, @Context HttpServletRequest hsr)
throws AuthorizationException, InterruptedException {
init();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
deleted file mode 100644
index 644ec90..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-
-import java.util.ArrayList;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for making scheduler configuration changes (supports adding,
- * removing, or updating a queue).
- */
-@XmlRootElement(name = "schedConf")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class QueueConfigsUpdateInfo {
-
- @XmlElement(name = "add")
- private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "remove")
- private ArrayList<String> removeQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "update")
- private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
-
- public QueueConfigsUpdateInfo() {
- // JAXB needs this
- }
-
- public ArrayList<QueueConfigInfo> getAddQueueInfo() {
- return addQueueInfo;
- }
-
- public ArrayList<String> getRemoveQueueInfo() {
- return removeQueueInfo;
- }
-
- public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
- return updateQueueInfo;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
new file mode 100644
index 0000000..b7c585e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue, as well as global scheduler conf changes).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedConfUpdateInfo {
+
+ @XmlElement(name = "add-queue")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove-queue")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update-queue")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ private HashMap<String, String> global = new HashMap<>();
+
+ public SchedConfUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+
+ @XmlElementWrapper(name = "global-updates")
+ public HashMap<String, String> getGlobalParams() {
+ return global;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
index 4016dcf..0f5a3d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -77,6 +77,7 @@ public class TestConfigurationMutationACLPolicies {
.thenReturn(false);
when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
}
+
@Test
public void testDefaultPolicy() {
Configuration conf = new Configuration();
@@ -98,7 +99,7 @@ public class TestConfigurationMutationACLPolicies {
ConfigurationMutationACLPolicy.class);
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
updateInfo.getUpdateQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -114,7 +115,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -130,7 +131,7 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Add root.b.b1.b11. Should check ACL of root.b queue.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
updateInfo.getAddQueueInfo().add(configInfo);
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
@@ -146,9 +147,26 @@ public class TestConfigurationMutationACLPolicies {
policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
policy.init(conf, rmContext);
// Remove root.b.b1.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b.b1");
assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
}
+
+ @Test
+ public void testQueueAdminPolicyGlobal() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertTrue(policy.isMutationAllowed(BAD_USER, updateInfo));
+ updateInfo.getGlobalParams().put("globalKey", "globalValue");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 13229b1..3216781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -47,8 +47,8 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private QueueConfigsUpdateInfo goodUpdate;
- private QueueConfigsUpdateInfo badUpdate;
+ private SchedConfUpdateInfo goodUpdate;
+ private SchedConfUpdateInfo badUpdate;
private CapacityScheduler cs;
private static final UserGroupInformation TEST_USER = UserGroupInformation
@@ -62,14 +62,14 @@ public class TestMutableCSConfigurationProvider {
when(cs.getConfiguration()).thenReturn(
new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new QueueConfigsUpdateInfo();
+ goodUpdate = new SchedConfUpdateInfo();
Map<String, String> goodUpdateMap = new HashMap<>();
goodUpdateMap.put("goodKey", "goodVal");
QueueConfigInfo goodUpdateInfo = new
QueueConfigInfo("root.a", goodUpdateMap);
goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
- badUpdate = new QueueConfigsUpdateInfo();
+ badUpdate = new SchedConfUpdateInfo();
Map<String, String> badUpdateMap = new HashMap<>();
badUpdateMap.put("badKey", "badVal");
QueueConfigInfo badUpdateInfo = new
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a979ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index d149055..5fbe36f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -162,7 +162,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add parent queue root.d with two children d1 and d2.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> d1Capacity = new HashMap<>();
d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
@@ -181,9 +181,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getAddQueueInfo().add(d);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -205,7 +205,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> dCapacity = new HashMap<>();
dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
Map<String, String> bCapacity = new HashMap<>();
@@ -216,9 +216,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -238,13 +238,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a.a2");
// Remove root.a.a2
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a.a2");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -263,13 +263,13 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.c", "root.c.c1");
// Remove root.c (parent queue)
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.c");
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -288,7 +288,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.a", "root.a.a1", "root.a.a2");
// Remove root.a (parent queue) with capacity 25
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.a");
// Set root.b capacity to 100
@@ -298,9 +298,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -320,7 +320,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
stopQueue("root.b", "root.c", "root.c.c1");
// Remove root.b and root.c
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
updateInfo.getRemoveQueueInfo().add("root.b");
updateInfo.getRemoveQueueInfo().add("root.c");
Map<String, String> aCapacity = new HashMap<>();
@@ -330,9 +330,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(configInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -348,7 +348,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Set state of queues to STOPPED.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> stoppedParam = new HashMap<>();
stoppedParam.put(CapacitySchedulerConfiguration.STATE,
QueueState.STOPPED.toString());
@@ -358,9 +358,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -378,7 +378,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update config value.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
"0.2");
@@ -393,9 +393,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
0.001f);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -411,9 +411,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -431,7 +431,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
ClientResponse response;
// Update root.a and root.b capacity to 50.
- QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
Map<String, String> updateParam = new HashMap<>();
updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
@@ -441,9 +441,9 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("queues").queryParam("user.name", userName)
+ .path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -453,6 +453,47 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
}
+ @Test
+ public void testGlobalConfChange() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set maximum-applications to 30000.
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", "30000");
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(30000, newCSConf.getMaximumSystemApplications());
+
+ updateInfo.getGlobalParams().put(CapacitySchedulerConfiguration.PREFIX +
+ "maximum-applications", null);
+ // Unset maximum-applications. Should be set to default.
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("sched-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS,
+ newCSConf.getMaximumSystemApplications());
+ }
+
@Override
@After
public void tearDown() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: Add 3.0.0-beta1 CHANGES and
RELEASENOTES.
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b34b3ff9/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/CHANGES.3.0.0-beta1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/CHANGES.3.0.0-beta1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/CHANGES.3.0.0-beta1.md
new file mode 100644
index 0000000..3ddf2ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/CHANGES.3.0.0-beta1.md
@@ -0,0 +1,646 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop Changelog
+
+## Release 3.0.0-beta1 - 2017-09-28
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-14539](https://issues.apache.org/jira/browse/HADOOP-14539) | Move commons logging APIs over to slf4j in hadoop-common | Major | . | Akira Ajisaka | Wenxin He |
+| [HDFS-12206](https://issues.apache.org/jira/browse/HDFS-12206) | Rename the split EC / replicated block metrics | Major | metrics | Andrew Wang | Andrew Wang |
+| [HADOOP-13595](https://issues.apache.org/jira/browse/HADOOP-13595) | Rework hadoop\_usage to be broken up by clients/daemons/etc. | Blocker | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-6984](https://issues.apache.org/jira/browse/HDFS-6984) | Serialize FileStatus via protobuf | Major | . | Colin P. McCabe | Chris Douglas |
+| [YARN-6961](https://issues.apache.org/jira/browse/YARN-6961) | Remove commons-logging dependency from hadoop-yarn-server-applicationhistoryservice module | Minor | build | Akira Ajisaka | Yeliang Cang |
+| [HDFS-11957](https://issues.apache.org/jira/browse/HDFS-11957) | Enable POSIX ACL inheritance by default | Major | security | John Zhuge | John Zhuge |
+| [MAPREDUCE-6870](https://issues.apache.org/jira/browse/MAPREDUCE-6870) | Add configuration for MR job to finish when all reducers are complete (even with unfinished mappers) | Major | . | Zhe Zhang | Peter Bacsko |
+| [HADOOP-14260](https://issues.apache.org/jira/browse/HADOOP-14260) | Configuration.dumpConfiguration should redact sensitive information | Major | conf, security | Vihang Karajgaonkar | John Zhuge |
+| [HADOOP-14726](https://issues.apache.org/jira/browse/HADOOP-14726) | Mark FileStatus::isDir as final | Minor | fs | Chris Douglas | Chris Douglas |
+| [HDFS-12303](https://issues.apache.org/jira/browse/HDFS-12303) | Change default EC cell size to 1MB for better performance | Blocker | . | Wei Zhou | Wei Zhou |
+| [HDFS-12258](https://issues.apache.org/jira/browse/HDFS-12258) | ec -listPolicies should list all policies in system, no matter it's enabled or disabled | Major | . | SammiChen | Wei Zhou |
+| [MAPREDUCE-6892](https://issues.apache.org/jira/browse/MAPREDUCE-6892) | Issues with the count of failed/killed tasks in the jhist file | Major | client, jobhistoryserver | Peter Bacsko | Peter Bacsko |
+| [HADOOP-14414](https://issues.apache.org/jira/browse/HADOOP-14414) | Calling maven-site-plugin directly for docs profile is unnecessary | Minor | . | Andras Bokor | Andras Bokor |
+| [HDFS-12218](https://issues.apache.org/jira/browse/HDFS-12218) | Rename split EC / replicated block metrics in BlockManager | Blocker | erasure-coding, metrics | Andrew Wang | Andrew Wang |
+| [HADOOP-14847](https://issues.apache.org/jira/browse/HADOOP-14847) | Remove Guava Supplier and change to java Supplier in AMRMClient and AMRMClientAysnc | Blocker | . | Bharat Viswanadham | Bharat Viswanadham |
+| [HDFS-12414](https://issues.apache.org/jira/browse/HDFS-12414) | Ensure to use CLI command to enable/disable erasure coding policy | Major | . | SammiChen | SammiChen |
+| [HDFS-12438](https://issues.apache.org/jira/browse/HDFS-12438) | Rename dfs.datanode.ec.reconstruction.stripedblock.threads.size to dfs.datanode.ec.reconstruction.threads | Major | . | Andrew Wang | Andrew Wang |
+| [HADOOP-14738](https://issues.apache.org/jira/browse/HADOOP-14738) | Remove S3N and obsolete bits of S3A; rework docs | Blocker | fs/s3 | Steve Loughran | Steve Loughran |
+| [HDFS-7859](https://issues.apache.org/jira/browse/HDFS-7859) | Erasure Coding: Persist erasure coding policies in NameNode | Major | . | Kai Zheng | SammiChen |
+| [HDFS-12395](https://issues.apache.org/jira/browse/HDFS-12395) | Support erasure coding policy operations in namenode edit log | Major | erasure-coding | SammiChen | SammiChen |
+| [HADOOP-14670](https://issues.apache.org/jira/browse/HADOOP-14670) | Increase minimum cmake version for all platforms | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-12447](https://issues.apache.org/jira/browse/HDFS-12447) | Rename AddECPolicyResponse to AddErasureCodingPolicyResponse | Major | . | SammiChen | SammiChen |
+| [HDFS-7337](https://issues.apache.org/jira/browse/HDFS-7337) | Configurable and pluggable erasure codec and policy | Critical | erasure-coding | Zhe Zhang | SammiChen |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-6959](https://issues.apache.org/jira/browse/YARN-6959) | RM may allocate wrong AM Container for new attempt | Major | capacity scheduler, fairscheduler, scheduler | Yuqi Wang | Yuqi Wang |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-4161](https://issues.apache.org/jira/browse/YARN-4161) | Capacity Scheduler : Assign single or multiple containers per heart beat driven by configuration | Major | capacity scheduler | Mayank Bansal | Wei Yan |
+| [HADOOP-14560](https://issues.apache.org/jira/browse/HADOOP-14560) | Make HttpServer2 backlog size configurable | Major | common | Alexander Krasheninnikov | Alexander Krasheninnikov |
+| [HDFS-10899](https://issues.apache.org/jira/browse/HDFS-10899) | Add functionality to re-encrypt EDEKs | Major | encryption, kms | Xiao Chen | Xiao Chen |
+| [YARN-5355](https://issues.apache.org/jira/browse/YARN-5355) | YARN Timeline Service v.2: alpha 2 | Critical | timelineserver | Sangjin Lee | Vrushali C |
+| [HADOOP-13345](https://issues.apache.org/jira/browse/HADOOP-13345) | S3Guard: Improved Consistency for S3A | Major | fs/s3 | Chris Nauroth | Chris Nauroth |
+| [HADOOP-12077](https://issues.apache.org/jira/browse/HADOOP-12077) | Provide a multi-URI replication Inode for ViewFs | Major | fs | Gera Shegalov | Gera Shegalov |
+| [HDFS-7877](https://issues.apache.org/jira/browse/HDFS-7877) | Support maintenance state for datanodes | Major | datanode, namenode | Ming Ma | Ming Ma |
+| [YARN-2915](https://issues.apache.org/jira/browse/YARN-2915) | Enable YARN RM scale out via federation using multiple RM's | Major | nodemanager, resourcemanager | Sriram Rao | Subru Krishnan |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HDFS-12042](https://issues.apache.org/jira/browse/HDFS-12042) | Lazy initialize AbstractINodeDiffList#diffs for snapshots to reduce memory consumption | Major | . | Misha Dmitriev | Misha Dmitriev |
+| [HDFS-12078](https://issues.apache.org/jira/browse/HDFS-12078) | Add time unit to the description of property dfs.namenode.stale.datanode.interval in hdfs-default.xml | Minor | documentation, hdfs | Weiwei Yang | Weiwei Yang |
+| [YARN-6752](https://issues.apache.org/jira/browse/YARN-6752) | Display reserved resources in web UI per application | Major | fairscheduler | Abdullah Yousufi | Abdullah Yousufi |
+| [YARN-6746](https://issues.apache.org/jira/browse/YARN-6746) | SchedulerUtils.checkResourceRequestMatchingNodePartition() is dead code | Minor | scheduler | Daniel Templeton | Deepti Sawhney |
+| [YARN-6410](https://issues.apache.org/jira/browse/YARN-6410) | FSContext.scheduler should be final | Minor | fairscheduler | Daniel Templeton | Yeliang Cang |
+| [YARN-6764](https://issues.apache.org/jira/browse/YARN-6764) | Simplify the logic in FairScheduler#attemptScheduling | Trivial | fairscheduler | Yufei Gu | Yufei Gu |
+| [HADOOP-14443](https://issues.apache.org/jira/browse/HADOOP-14443) | Azure: Support retry and client side failover for authorization, SASKey and delegation token generation | Major | fs/azure | Santhosh G Nayak | Santhosh G Nayak |
+| [HADOOP-14535](https://issues.apache.org/jira/browse/HADOOP-14535) | wasb: implement high-performance random access and seek of block blobs | Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [HADOOP-14629](https://issues.apache.org/jira/browse/HADOOP-14629) | Improve exception checking in FileContext related JUnit tests | Major | fs, test | Andras Bokor | Andras Bokor |
+| [YARN-6689](https://issues.apache.org/jira/browse/YARN-6689) | PlacementRule should be configurable | Major | . | Jonathan Hung | Jonathan Hung |
+| [HDFS-12130](https://issues.apache.org/jira/browse/HDFS-12130) | Optimizing permission check for getContentSummary | Major | namenode | Chen Liang | Chen Liang |
+| [HDFS-12137](https://issues.apache.org/jira/browse/HDFS-12137) | DN dataset lock should be fair | Critical | datanode | Daryn Sharp | Daryn Sharp |
+| [HDFS-11786](https://issues.apache.org/jira/browse/HDFS-11786) | Add support to make copyFromLocal multi threaded | Major | hdfs | Mukul Kumar Singh | Mukul Kumar Singh |
+| [HDFS-12138](https://issues.apache.org/jira/browse/HDFS-12138) | Remove redundant 'public' modifiers from BlockCollection | Trivial | namenode | Chen Liang | Chen Liang |
+| [HADOOP-14640](https://issues.apache.org/jira/browse/HADOOP-14640) | Azure: Support affinity for service running on localhost and reuse SPNEGO hadoop.auth cookie for authorization, SASKey and delegation token generation | Major | fs/azure | Santhosh G Nayak | Santhosh G Nayak |
+| [YARN-6778](https://issues.apache.org/jira/browse/YARN-6778) | In ResourceWeights, weights and setWeights() should be final | Minor | scheduler | Daniel Templeton | Daniel Templeton |
+| [HDFS-12067](https://issues.apache.org/jira/browse/HDFS-12067) | Correct dfsadmin commands usage message to reflects IPC port | Major | . | steven-wugang | steven-wugang |
+| [HADOOP-14666](https://issues.apache.org/jira/browse/HADOOP-14666) | Tests use assertTrue(....equals(...)) instead of assertEquals() | Minor | test | Daniel Templeton | Daniel Templeton |
+| [HADOOP-14659](https://issues.apache.org/jira/browse/HADOOP-14659) | UGI getShortUserName does not need to search the Subject | Major | common | Daryn Sharp | Daryn Sharp |
+| [HADOOP-14557](https://issues.apache.org/jira/browse/HADOOP-14557) | Document HADOOP-8143 (Change distcp to have -pb on by default) | Trivial | . | Wei-Chiu Chuang | Bharat Viswanadham |
+| [YARN-6768](https://issues.apache.org/jira/browse/YARN-6768) | Improve performance of yarn api record toString and fromString | Major | . | Jonathan Eagles | Jonathan Eagles |
+| [YARN-6779](https://issues.apache.org/jira/browse/YARN-6779) | DominantResourceFairnessPolicy.DominantResourceFairnessComparator.calculateShares() should be @VisibleForTesting | Trivial | fairscheduler | Daniel Templeton | Yeliang Cang |
+| [YARN-6845](https://issues.apache.org/jira/browse/YARN-6845) | Variable scheduler of FSLeafQueue duplicates the one of its parent FSQueue. | Trivial | fairscheduler | Yufei Gu | Yufei Gu |
+| [HADOOP-14597](https://issues.apache.org/jira/browse/HADOOP-14597) | Native compilation broken with OpenSSL-1.1.0 because EVP\_CIPHER\_CTX has been made opaque | Major | . | Ravi Prakash | Ravi Prakash |
+| [HDFS-12193](https://issues.apache.org/jira/browse/HDFS-12193) | Fix style issues in HttpFS tests | Trivial | httpfs | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [HADOOP-14681](https://issues.apache.org/jira/browse/HADOOP-14681) | Remove MockitoMaker class | Major | test | Andras Bokor | Andras Bokor |
+| [HDFS-12143](https://issues.apache.org/jira/browse/HDFS-12143) | Improve performance of getting and removing inode features | Major | namenode | Daryn Sharp | Daryn Sharp |
+| [HDFS-12171](https://issues.apache.org/jira/browse/HDFS-12171) | Reduce IIP object allocations for inode lookup | Major | namenode | Daryn Sharp | Daryn Sharp |
+| [HDFS-12190](https://issues.apache.org/jira/browse/HDFS-12190) | Enable 'hdfs dfs -stat' to display access time | Major | hdfs, shell | Yongjun Zhang | Yongjun Zhang |
+| [YARN-6864](https://issues.apache.org/jira/browse/YARN-6864) | FSPreemptionThread cleanup for readability | Minor | fairscheduler | Daniel Templeton | Daniel Templeton |
+| [HADOOP-14455](https://issues.apache.org/jira/browse/HADOOP-14455) | ViewFileSystem#rename should support be supported within same nameservice with different mountpoints | Major | viewfs | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-14690](https://issues.apache.org/jira/browse/HADOOP-14690) | RetryInvocationHandler$RetryInfo should override toString() | Minor | . | Akira Ajisaka | Yeliang Cang |
+| [HADOOP-14709](https://issues.apache.org/jira/browse/HADOOP-14709) | Fix checkstyle warnings in ContractTestUtils | Minor | test | Steve Loughran | Thomas Marquardt |
+| [MAPREDUCE-6914](https://issues.apache.org/jira/browse/MAPREDUCE-6914) | Tests use assertTrue(....equals(...)) instead of assertEquals() | Minor | test | Daniel Templeton | Daniel Templeton |
+| [YARN-6832](https://issues.apache.org/jira/browse/YARN-6832) | Tests use assertTrue(....equals(...)) instead of assertEquals() | Minor | test | Daniel Templeton | Daniel Templeton |
+| [HADOOP-14706](https://issues.apache.org/jira/browse/HADOOP-14706) | Adding a helper method to determine whether a log is Log4j implement | Minor | util | Wenxin He | Wenxin He |
+| [HADOOP-14471](https://issues.apache.org/jira/browse/HADOOP-14471) | Upgrade Jetty to latest 9.3 version | Major | . | John Zhuge | John Zhuge |
+| [HDFS-12251](https://issues.apache.org/jira/browse/HDFS-12251) | Add document for StreamCapabilities | Major | . | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HDFS-12224](https://issues.apache.org/jira/browse/HDFS-12224) | Add tests to TestJournalNodeSync for sync after JN downtime | Major | hdfs | Hanisha Koneru | Hanisha Koneru |
+| [YARN-6802](https://issues.apache.org/jira/browse/YARN-6802) | Add Max AM Resource and AM Resource Usage to Leaf Queue View in FairScheduler WebUI | Major | fairscheduler | YunFan Zhou | YunFan Zhou |
+| [HDFS-12036](https://issues.apache.org/jira/browse/HDFS-12036) | Add audit log for some erasure coding operations | Major | namenode | Wei-Chiu Chuang | Huafeng Wang |
+| [HDFS-12264](https://issues.apache.org/jira/browse/HDFS-12264) | DataNode uses a deprecated method IoUtils#cleanup. | Major | . | Ajay Kumar | Ajay Kumar |
+| [YARN-6811](https://issues.apache.org/jira/browse/YARN-6811) | [ATS1.5] All history logs should be kept under its own User Directory. | Major | timelineclient, timelineserver | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-6879](https://issues.apache.org/jira/browse/YARN-6879) | TestLeafQueue.testDRFUserLimits() has commented out code | Trivial | capacity scheduler, test | Daniel Templeton | Angela Wang |
+| [MAPREDUCE-6923](https://issues.apache.org/jira/browse/MAPREDUCE-6923) | Optimize MapReduce Shuffle I/O for small partitions | Major | . | Robert Schmidtke | Robert Schmidtke |
+| [HDFS-12287](https://issues.apache.org/jira/browse/HDFS-12287) | Remove a no-longer applicable TODO comment in DatanodeManager | Trivial | namenode | Chen Liang | Chen Liang |
+| [YARN-6952](https://issues.apache.org/jira/browse/YARN-6952) | Enable scheduling monitor in FS | Major | fairscheduler, resourcemanager | Yufei Gu | Yufei Gu |
+| [YARN-6884](https://issues.apache.org/jira/browse/YARN-6884) | AllocationFileLoaderService.loadQueue() has an if without braces | Trivial | fairscheduler | Daniel Templeton | weiyuan |
+| [YARN-6882](https://issues.apache.org/jira/browse/YARN-6882) | AllocationFileLoaderService.reloadAllocations() should use the diamond operator | Trivial | fairscheduler | Daniel Templeton | Larry Lo |
+| [HADOOP-14627](https://issues.apache.org/jira/browse/HADOOP-14627) | Support MSI and DeviceCode token provider in ADLS | Major | fs/adl | Atul Sikaria | Atul Sikaria |
+| [HDFS-12221](https://issues.apache.org/jira/browse/HDFS-12221) | Replace xerces in XmlEditsVisitor | Major | . | Lei (Eddy) Xu | Ajay Kumar |
+| [HADOOP-14741](https://issues.apache.org/jira/browse/HADOOP-14741) | Refactor curator based ZooKeeper communication into common library | Major | . | Subru Krishnan | Íñigo Goiri |
+| [HDFS-12162](https://issues.apache.org/jira/browse/HDFS-12162) | Update listStatus document to describe the behavior when the argument is a file | Major | hdfs, httpfs | Yongjun Zhang | Ajay Kumar |
+| [YARN-6881](https://issues.apache.org/jira/browse/YARN-6881) | LOG is unused in AllocationConfiguration | Major | fairscheduler | Daniel Templeton | weiyuan |
+| [YARN-6917](https://issues.apache.org/jira/browse/YARN-6917) | Queue path is recomputed from scratch on every allocation | Minor | capacityscheduler | Jason Lowe | Eric Payne |
+| [HADOOP-14673](https://issues.apache.org/jira/browse/HADOOP-14673) | Remove leftover hadoop\_xml\_escape from functions | Major | scripts | Allen Wittenauer | Ajay Kumar |
+| [HADOOP-14662](https://issues.apache.org/jira/browse/HADOOP-14662) | Update azure-storage sdk to version 5.4.0 | Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [HDFS-12301](https://issues.apache.org/jira/browse/HDFS-12301) | NN File Browser UI: Navigate to a path when enter is pressed | Trivial | ui | Ravi Prakash | Ravi Prakash |
+| [HDFS-12269](https://issues.apache.org/jira/browse/HDFS-12269) | Better to return a Map rather than HashMap in getErasureCodingCodecs | Minor | erasure-coding | Huafeng Wang | Huafeng Wang |
+| [YARN-3254](https://issues.apache.org/jira/browse/YARN-3254) | HealthReport should include disk full information | Major | nodemanager | Akira Ajisaka | Suma Shivaprasad |
+| [HDFS-12072](https://issues.apache.org/jira/browse/HDFS-12072) | Provide fairness between EC and non-EC recovery tasks. | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HDFS-12117](https://issues.apache.org/jira/browse/HDFS-12117) | HttpFS does not seem to support SNAPSHOT related methods for WebHDFS REST Interface | Major | httpfs | Wellington Chevreuil | Wellington Chevreuil |
+| [HADOOP-14705](https://issues.apache.org/jira/browse/HADOOP-14705) | Add batched interface reencryptEncryptedKeys to KMS | Major | kms | Xiao Chen | Xiao Chen |
+| [YARN-7053](https://issues.apache.org/jira/browse/YARN-7053) | Move curator transaction support to ZKCuratorManager | Major | . | Jonathan Hung | Jonathan Hung |
+| [HADOOP-14251](https://issues.apache.org/jira/browse/HADOOP-14251) | Credential provider should handle property key deprecation | Critical | security | John Zhuge | John Zhuge |
+| [YARN-7049](https://issues.apache.org/jira/browse/YARN-7049) | FSAppAttempt preemption related fields have confusing names | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla |
+| [HDFS-11986](https://issues.apache.org/jira/browse/HDFS-11986) | Dfsadmin should report erasure coding related information separately | Major | erasure-coding, hdfs | Manoj Govindassamy | Manoj Govindassamy |
+| [YARN-6999](https://issues.apache.org/jira/browse/YARN-6999) | Add log about how to solve Error: Could not find or load main class org.apache.hadoop.mapreduce.v2.app.MRAppMaster | Minor | documentation, security | Linlin Zhou | Linlin Zhou |
+| [YARN-7037](https://issues.apache.org/jira/browse/YARN-7037) | Optimize data transfer with zero-copy approach for containerlogs REST API in NMWebServices | Major | nodemanager | Tao Yang | Tao Yang |
+| [HDFS-12356](https://issues.apache.org/jira/browse/HDFS-12356) | Unit test for JournalNode sync during Rolling Upgrade | Major | ha | Hanisha Koneru | Hanisha Koneru |
+| [YARN-6780](https://issues.apache.org/jira/browse/YARN-6780) | ResourceWeights.toString() cleanup | Minor | scheduler | Daniel Templeton | weiyuan |
+| [YARN-6721](https://issues.apache.org/jira/browse/YARN-6721) | container-executor should have stack checking | Critical | nodemanager, security | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-14781](https://issues.apache.org/jira/browse/HADOOP-14781) | Clarify that HADOOP\_CONF\_DIR shouldn't actually be set in hadoop-env.sh | Major | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-12380](https://issues.apache.org/jira/browse/HDFS-12380) | Simplify dataQueue.wait condition logical operation in DataStreamer::run() | Major | hdfs-client | liaoyuxiangqin | liaoyuxiangqin |
+| [HDFS-12300](https://issues.apache.org/jira/browse/HDFS-12300) | Audit-log delegation token related operations | Major | namenode | Xiao Chen | Xiao Chen |
+| [YARN-7022](https://issues.apache.org/jira/browse/YARN-7022) | Improve click interaction in queue topology in new YARN UI | Major | yarn-ui-v2 | Abdullah Yousufi | Abdullah Yousufi |
+| [HDFS-12182](https://issues.apache.org/jira/browse/HDFS-12182) | BlockManager.metaSave does not distinguish between "under replicated" and "missing" blocks | Trivial | hdfs | Wellington Chevreuil | Wellington Chevreuil |
+| [HADOOP-14688](https://issues.apache.org/jira/browse/HADOOP-14688) | Intern strings in KeyVersion and EncryptedKeyVersion | Major | kms | Xiao Chen | Xiao Chen |
+| [HDFS-12377](https://issues.apache.org/jira/browse/HDFS-12377) | Refactor TestReadStripedFileWithDecoding to avoid test timeouts | Major | erasure-coding | Andrew Wang | Andrew Wang |
+| [HADOOP-14827](https://issues.apache.org/jira/browse/HADOOP-14827) | Allow StopWatch to accept a Timer parameter for tests | Minor | common, test | Erik Krogen | Erik Krogen |
+| [HDFS-12131](https://issues.apache.org/jira/browse/HDFS-12131) | Add some of the FSNamesystem JMX values as metrics | Minor | hdfs, namenode | Erik Krogen | Erik Krogen |
+| [HDFS-12402](https://issues.apache.org/jira/browse/HDFS-12402) | Refactor ErasureCodingPolicyManager and related codes | Major | erasure-coding | SammiChen | SammiChen |
+| [YARN-6930](https://issues.apache.org/jira/browse/YARN-6930) | Admins should be able to explicitly enable specific LinuxContainerRuntime in the NodeManager | Major | nodemanager | Vinod Kumar Vavilapalli | Shane Kumpf |
+| [HADOOP-14844](https://issues.apache.org/jira/browse/HADOOP-14844) | Remove requirement to specify TenantGuid for MSI Token Provider | Major | fs/adl | Atul Sikaria | Atul Sikaria |
+| [YARN-6799](https://issues.apache.org/jira/browse/YARN-6799) | Remove the duplicated code in CGroupsHandlerImp.java | Trivial | nodemanager | Yufei Gu | weiyuan |
+| [HADOOP-14520](https://issues.apache.org/jira/browse/HADOOP-14520) | WASB: Block compaction for Azure Block Blobs | Major | fs/azure | Georgi Chalakov | Georgi Chalakov |
+| [HADOOP-14839](https://issues.apache.org/jira/browse/HADOOP-14839) | DistCp log output should contain copied and deleted files and directories | Major | tools/distcp | Konstantin Shaposhnikov | Yiqun Lin |
+| [YARN-7132](https://issues.apache.org/jira/browse/YARN-7132) | FairScheduler.initScheduler() contains a surprising unary plus | Minor | fairscheduler | Daniel Templeton | Yeliang Cang |
+| [HADOOP-14843](https://issues.apache.org/jira/browse/HADOOP-14843) | Improve FsPermission symbolic parsing unit test coverage | Minor | fs | Jason Lowe | Bharat Viswanadham |
+| [YARN-7057](https://issues.apache.org/jira/browse/YARN-7057) | FSAppAttempt#getResourceUsage doesn't need to consider resources queued for preemption | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla |
+| [HDFS-12412](https://issues.apache.org/jira/browse/HDFS-12412) | Change ErasureCodingWorker.stripedReadPool to cached thread pool | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-14804](https://issues.apache.org/jira/browse/HADOOP-14804) | correct wrong parameters format order in core-default.xml | Trivial | . | Chen Hongfei | Chen Hongfei |
+| [HDFS-12409](https://issues.apache.org/jira/browse/HDFS-12409) | Add metrics of execution time of different stages in EC recovery task | Minor | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-14864](https://issues.apache.org/jira/browse/HADOOP-14864) | FSDataInputStream#unbuffer UOE should include stream class name | Minor | fs | John Zhuge | Bharat Viswanadham |
+| [HADOOP-14869](https://issues.apache.org/jira/browse/HADOOP-14869) | Upgrade apache kerby verion to v1.0.1 | Major | . | Wei Zhou | Wei Zhou |
+| [MAPREDUCE-6956](https://issues.apache.org/jira/browse/MAPREDUCE-6956) | FileOutputCommitter to gain abstract superclass PathOutputCommitter | Minor | mrv2 | Steve Loughran | Steve Loughran |
+| [MAPREDUCE-6954](https://issues.apache.org/jira/browse/MAPREDUCE-6954) | Disable erasure coding for files that are uploaded to the MR staging area | Major | client | Peter Bacsko | Peter Bacsko |
+| [HDFS-12349](https://issues.apache.org/jira/browse/HDFS-12349) | Improve log message when it could not alloc enough blocks for EC | Minor | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HDFS-12441](https://issues.apache.org/jira/browse/HDFS-12441) | Suppress UnresolvedPathException in namenode log | Minor | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-13714](https://issues.apache.org/jira/browse/HADOOP-13714) | Tighten up our compatibility guidelines for Hadoop 3 | Blocker | documentation | Karthik Kambatla | Daniel Templeton |
+| [HDFS-12472](https://issues.apache.org/jira/browse/HDFS-12472) | Add JUNIT timeout to TestBlockStatsMXBean | Minor | . | Lei (Eddy) Xu | Bharat Viswanadham |
+| [HDFS-12460](https://issues.apache.org/jira/browse/HDFS-12460) | Make addErasureCodingPolicy an idempotent operation | Major | erasure-coding | SammiChen | SammiChen |
+| [HDFS-12479](https://issues.apache.org/jira/browse/HDFS-12479) | Some misuses of lock in DFSStripedOutputStream | Minor | erasure-coding | Huafeng Wang | Huafeng Wang |
+| [MAPREDUCE-6958](https://issues.apache.org/jira/browse/MAPREDUCE-6958) | Shuffle audit logger should log size of shuffle transfer | Minor | . | Jason Lowe | Jason Lowe |
+| [HDFS-12444](https://issues.apache.org/jira/browse/HDFS-12444) | Reduce runtime of TestWriteReadStripedFile | Major | erasure-coding, test | Andrew Wang | Huafeng Wang |
+| [HDFS-12445](https://issues.apache.org/jira/browse/HDFS-12445) | Correct spellings of choosen to chosen. | Trivial | . | hu xiaodong | hu xiaodong |
+| [HADOOP-7308](https://issues.apache.org/jira/browse/HADOOP-7308) | Remove unused TaskLogAppender configurations from log4j.properties | Major | conf | Todd Lipcon | Todd Lipcon |
+| [HDFS-12496](https://issues.apache.org/jira/browse/HDFS-12496) | Make QuorumJournalManager timeout properties configurable | Major | . | Ajay Kumar | Ajay Kumar |
+| [HDFS-12530](https://issues.apache.org/jira/browse/HDFS-12530) | Processor argument in Offline Image Viewer should be case insensitive | Minor | tools | Hanisha Koneru | Hanisha Koneru |
+| [HDFS-12304](https://issues.apache.org/jira/browse/HDFS-12304) | Remove unused parameter from FsDatasetImpl#addVolume | Minor | . | Chen Liang | Chen Liang |
+| [YARN-65](https://issues.apache.org/jira/browse/YARN-65) | Reduce RM app memory footprint once app has completed | Major | resourcemanager | Jason Lowe | Manikandan R |
+
+
+### BUG FIXES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-6093](https://issues.apache.org/jira/browse/YARN-6093) | Minor bugs with AMRMtoken renewal and state store availability when using FederationRMFailoverProxyProvider during RM failover | Minor | amrmproxy, federation | Botong Huang | Botong Huang |
+| [YARN-6370](https://issues.apache.org/jira/browse/YARN-6370) | Properly handle rack requests for non-active subclusters in LocalityMulticastAMRMProxyPolicy | Minor | . | Botong Huang | Botong Huang |
+| [HDFS-12043](https://issues.apache.org/jira/browse/HDFS-12043) | Add counters for block re-replication | Major | . | Chen Liang | Chen Liang |
+| [YARN-6694](https://issues.apache.org/jira/browse/YARN-6694) | Add certain envs to the default yarn.nodemanager.env-whitelist | Major | . | Jian He | Jian He |
+| [MAPREDUCE-6905](https://issues.apache.org/jira/browse/MAPREDUCE-6905) | Fix meaningless operations in TestDFSIO in some situation. | Major | tools/rumen | LiXin Ge | LiXin Ge |
+| [HDFS-12079](https://issues.apache.org/jira/browse/HDFS-12079) | Description of dfs.block.invalidate.limit is incorrect in hdfs-default.xml | Minor | documentation | Weiwei Yang | Weiwei Yang |
+| [HADOOP-13414](https://issues.apache.org/jira/browse/HADOOP-13414) | Hide Jetty Server version header in HTTP responses | Major | security | Vinayakumar B | Surendra Singh Lilhore |
+| [HDFS-12089](https://issues.apache.org/jira/browse/HDFS-12089) | Fix ambiguous NN retry log message | Major | webhdfs | Eric Badger | Eric Badger |
+| [HADOOP-14608](https://issues.apache.org/jira/browse/HADOOP-14608) | KMS JMX servlet path not backwards compatible | Minor | kms | John Zhuge | John Zhuge |
+| [YARN-6708](https://issues.apache.org/jira/browse/YARN-6708) | Nodemanager container crash after ext3 folder limit | Critical | . | Bibin A Chundatt | Bibin A Chundatt |
+| [HADOOP-14563](https://issues.apache.org/jira/browse/HADOOP-14563) | LoadBalancingKMSClientProvider#warmUpEncryptedKeys swallows IOException | Major | . | Rushabh S Shah | Rushabh S Shah |
+| [MAPREDUCE-6246](https://issues.apache.org/jira/browse/MAPREDUCE-6246) | DBOutputFormat.java appending extra semicolon to query which is incompatible with DB2 | Major | mrv1, mrv2 | ramtin | Gergely Novák |
+| [HADOOP-14634](https://issues.apache.org/jira/browse/HADOOP-14634) | Remove jline from main Hadoop pom.xml | Major | . | Ray Chiang | Ray Chiang |
+| [YARN-6428](https://issues.apache.org/jira/browse/YARN-6428) | Queue AM limit is not honored in CS always | Major | . | Bibin A Chundatt | Bibin A Chundatt |
+| [YARN-6770](https://issues.apache.org/jira/browse/YARN-6770) | [Docs] A small mistake in the example of TimelineClient | Trivial | docs | Jinjiang Ling | Jinjiang Ling |
+| [HADOOP-10829](https://issues.apache.org/jira/browse/HADOOP-10829) | Iteration on CredentialProviderFactory.serviceLoader is thread-unsafe | Major | security | Benoy Antony | Benoy Antony |
+| [HDFS-12085](https://issues.apache.org/jira/browse/HDFS-12085) | Reconfigure namenode heartbeat interval fails if the interval was set with time unit | Minor | hdfs, tools | Weiwei Yang | Weiwei Yang |
+| [HDFS-12052](https://issues.apache.org/jira/browse/HDFS-12052) | Set SWEBHDFS delegation token kind when ssl is enabled in HttpFS | Major | httpfs, webhdfs | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [HDFS-12114](https://issues.apache.org/jira/browse/HDFS-12114) | Consistent HttpFS property names | Major | httpfs | John Zhuge | John Zhuge |
+| [HADOOP-14581](https://issues.apache.org/jira/browse/HADOOP-14581) | Restrict setOwner to list of user when security is enabled in wasb | Major | fs/azure | Varada Hemeswari | Varada Hemeswari |
+| [YARN-6809](https://issues.apache.org/jira/browse/YARN-6809) | Fix typo in ResourceManagerHA.md | Trivial | documentation | Akira Ajisaka | Yeliang Cang |
+| [YARN-6797](https://issues.apache.org/jira/browse/YARN-6797) | TimelineWriter does not fully consume the POST response | Major | timelineclient | Jason Lowe | Jason Lowe |
+| [HDFS-11502](https://issues.apache.org/jira/browse/HDFS-11502) | Datanode UI should display hostname based on JMX bean instead of window.location.hostname | Major | hdfs | Jeffrey E Rodriguez | Jeffrey E Rodriguez |
+| [HADOOP-14646](https://issues.apache.org/jira/browse/HADOOP-14646) | FileContextMainOperationsBaseTest#testListStatusFilterWithSomeMatches never runs | Minor | test | Andras Bokor | Andras Bokor |
+| [YARN-6654](https://issues.apache.org/jira/browse/YARN-6654) | RollingLevelDBTimelineStore backwards incompatible after fst upgrade | Blocker | . | Jonathan Eagles | Jonathan Eagles |
+| [YARN-6805](https://issues.apache.org/jira/browse/YARN-6805) | NPE in LinuxContainerExecutor due to null PrivilegedOperationException exit code | Major | nodemanager | Jason Lowe | Jason Lowe |
+| [YARN-6792](https://issues.apache.org/jira/browse/YARN-6792) | Incorrect XML convertion in NodeIDsInfo and LabelsToNodesInfo | Blocker | resourcemanager | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-6769](https://issues.apache.org/jira/browse/YARN-6769) | Make schedulables without demand less needy in FairSharePolicy#compare | Major | fairscheduler | YunFan Zhou | YunFan Zhou |
+| [YARN-6759](https://issues.apache.org/jira/browse/YARN-6759) | Fix TestRMRestart.testRMRestartWaitForPreviousAMToFinish failure | Major | . | Naganarasimha G R | Naganarasimha G R |
+| [YARN-3260](https://issues.apache.org/jira/browse/YARN-3260) | AM attempt fail to register before RM processes launch event | Critical | resourcemanager | Jason Lowe | Bibin A Chundatt |
+| [HDFS-12140](https://issues.apache.org/jira/browse/HDFS-12140) | Remove BPOfferService lock contention to get block pool id | Critical | datanode | Daryn Sharp | Daryn Sharp |
+| [YARN-6625](https://issues.apache.org/jira/browse/YARN-6625) | yarn application -list returns a tracking URL for AM that doesn't work in secured and HA environment | Major | amrmproxy | Yufei Gu | Yufei Gu |
+| [HDFS-12112](https://issues.apache.org/jira/browse/HDFS-12112) | TestBlockManager#testBlockManagerMachinesArray sometimes fails with NPE | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [YARN-6714](https://issues.apache.org/jira/browse/YARN-6714) | IllegalStateException while handling APP\_ATTEMPT\_REMOVED event when async-scheduling enabled in CapacityScheduler | Major | . | Tao Yang | Tao Yang |
+| [MAPREDUCE-6889](https://issues.apache.org/jira/browse/MAPREDUCE-6889) | Add Job#close API to shutdown MR client services. | Major | . | Rohith Sharma K S | Rohith Sharma K S |
+| [MAPREDUCE-6910](https://issues.apache.org/jira/browse/MAPREDUCE-6910) | MapReduceTrackingUriPlugin can not return the right URI of history server with HTTPS | Major | jobhistoryserver | Lantao Jin | Lantao Jin |
+| [HDFS-12154](https://issues.apache.org/jira/browse/HDFS-12154) | Incorrect javadoc description in StorageLocationChecker#check | Major | . | Nandakumar | Nandakumar |
+| [YARN-6798](https://issues.apache.org/jira/browse/YARN-6798) | Fix NM startup failure with old state store due to version mismatch | Major | nodemanager | Ray Chiang | Botong Huang |
+| [HADOOP-14637](https://issues.apache.org/jira/browse/HADOOP-14637) | GenericTestUtils.waitFor needs to check condition again after max wait time | Major | . | Daniel Templeton | Daniel Templeton |
+| [YARN-6819](https://issues.apache.org/jira/browse/YARN-6819) | Application report fails if app rejected due to nodesize | Major | . | Bibin A Chundatt | Bibin A Chundatt |
+| [HADOOP-14669](https://issues.apache.org/jira/browse/HADOOP-14669) | GenericTestUtils.waitFor should use monotonic time | Trivial | test | Jason Lowe | Daniel Templeton |
+| [HDFS-12133](https://issues.apache.org/jira/browse/HDFS-12133) | Correct ContentSummaryComputationContext Logger class name. | Minor | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-12158](https://issues.apache.org/jira/browse/HDFS-12158) | Secondary Namenode's web interface lack configs for X-FRAME-OPTIONS protection | Major | namenode | Mukul Kumar Singh | Mukul Kumar Singh |
+| [HDFS-12139](https://issues.apache.org/jira/browse/HDFS-12139) | HTTPFS liststatus returns incorrect pathSuffix for path of file | Major | httpfs | Yongjun Zhang | Yongjun Zhang |
+| [YARN-6837](https://issues.apache.org/jira/browse/YARN-6837) | Null LocalResource visibility or resource type can crash the nodemanager | Major | . | Jinjiang Ling | Jinjiang Ling |
+| [HDFS-11472](https://issues.apache.org/jira/browse/HDFS-11472) | Fix inconsistent replica size after a data pipeline failure | Critical | datanode | Wei-Chiu Chuang | Erik Krogen |
+| [HDFS-12166](https://issues.apache.org/jira/browse/HDFS-12166) | Do not deprecate HTTPFS\_TEMP | Minor | httpfs | John Zhuge | John Zhuge |
+| [HDFS-11742](https://issues.apache.org/jira/browse/HDFS-11742) | Improve balancer usability after HDFS-8818 | Blocker | . | Kihwal Lee | Kihwal Lee |
+| [HDFS-12094](https://issues.apache.org/jira/browse/HDFS-12094) | Log torrent when none isa-l EC is used. | Minor | erasure-coding | LiXin Ge | LiXin Ge |
+| [HDFS-12176](https://issues.apache.org/jira/browse/HDFS-12176) | dfsadmin shows DFS Used%: NaN% if the cluster has zero block. | Trivial | . | Wei-Chiu Chuang | Weiwei Yang |
+| [YARN-6844](https://issues.apache.org/jira/browse/YARN-6844) | AMRMClientImpl.checkNodeLabelExpression() has wrong error message | Minor | . | Daniel Templeton | Manikandan R |
+| [YARN-6150](https://issues.apache.org/jira/browse/YARN-6150) | TestContainerManagerSecurity tests for Yarn Server are flakey | Major | test | Daniel Sturman | Daniel Sturman |
+| [YARN-6307](https://issues.apache.org/jira/browse/YARN-6307) | Refactor FairShareComparator#compare | Major | fairscheduler | Yufei Gu | Yufei Gu |
+| [YARN-6102](https://issues.apache.org/jira/browse/YARN-6102) | RMActiveService context to be updated with new RMContext on failover | Critical | . | Ajith S | Rohith Sharma K S |
+| [HADOOP-14578](https://issues.apache.org/jira/browse/HADOOP-14578) | Bind IPC connections to kerberos UPN host for proxy users | Major | ipc | Daryn Sharp | Daryn Sharp |
+| [HDFS-11896](https://issues.apache.org/jira/browse/HDFS-11896) | Non-dfsUsed will be doubled on dead node re-registration | Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-14692](https://issues.apache.org/jira/browse/HADOOP-14692) | Upgrade Apache Rat | Trivial | build | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-12062](https://issues.apache.org/jira/browse/HDFS-12062) | removeErasureCodingPolicy needs super user permission | Critical | erasure-coding | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HDFS-12044](https://issues.apache.org/jira/browse/HDFS-12044) | Mismatch between BlockManager#maxReplicationStreams and ErasureCodingWorker.stripedReconstructionPool pool size causes slow and bursty recovery | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-14229](https://issues.apache.org/jira/browse/HADOOP-14229) | hadoop.security.auth\_to\_local example is incorrect in the documentation | Major | . | Andras Bokor | Andras Bokor |
+| [YARN-6870](https://issues.apache.org/jira/browse/YARN-6870) | Fix floating point inaccuracies in resource availability check in AllocationBasedResourceUtilizationTracker | Major | api, nodemanager | Brook Zhou | Brook Zhou |
+| [YARN-5728](https://issues.apache.org/jira/browse/YARN-5728) | TestMiniYarnClusterNodeUtilization.testUpdateNodeUtilization timeout | Major | test | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-14677](https://issues.apache.org/jira/browse/HADOOP-14677) | mvn clean compile fails | Major | build | Andras Bokor | Andras Bokor |
+| [HDFS-12082](https://issues.apache.org/jira/browse/HDFS-12082) | BlockInvalidateLimit value is incorrectly set after namenode heartbeat interval reconfigured | Major | hdfs, namenode | Weiwei Yang | Weiwei Yang |
+| [MAPREDUCE-6924](https://issues.apache.org/jira/browse/MAPREDUCE-6924) | Revert MAPREDUCE-6199 MAPREDUCE-6286 and MAPREDUCE-5875 | Major | . | Andrew Wang | Junping Du |
+| [HADOOP-14420](https://issues.apache.org/jira/browse/HADOOP-14420) | generateReports property is not applicable for maven-site-plugin:attach-descriptor goal | Minor | . | Andras Bokor | Andras Bokor |
+| [HADOOP-14644](https://issues.apache.org/jira/browse/HADOOP-14644) | Increase max heap size of Maven javadoc plugin | Major | test | Andras Bokor | Andras Bokor |
+| [HADOOP-14343](https://issues.apache.org/jira/browse/HADOOP-14343) | Wrong pid file name in error message when starting secure daemon | Minor | . | Andras Bokor | Andras Bokor |
+| [MAPREDUCE-6921](https://issues.apache.org/jira/browse/MAPREDUCE-6921) | TestUmbilicalProtocolWithJobToken#testJobTokenRpc fails | Major | . | Sonia Garudi | Sonia Garudi |
+| [HADOOP-14676](https://issues.apache.org/jira/browse/HADOOP-14676) | Wrong default value for "fs.df.interval" | Major | common, conf, fs | Konstantin Shvachko | xiangguang zheng |
+| [HADOOP-14701](https://issues.apache.org/jira/browse/HADOOP-14701) | Configuration can log misleading warnings about an attempt to override final parameter | Major | conf | Andrew Sherman | Andrew Sherman |
+| [YARN-5731](https://issues.apache.org/jira/browse/YARN-5731) | Preemption calculation is not accurate when reserved containers are present in queue. | Major | capacity scheduler | Sunil G | Wangda Tan |
+| [HADOOP-14683](https://issues.apache.org/jira/browse/HADOOP-14683) | FileStatus.compareTo binary compatible issue | Blocker | . | Sergey Shelukhin | Akira Ajisaka |
+| [HDFS-12107](https://issues.apache.org/jira/browse/HDFS-12107) | FsDatasetImpl#removeVolumes floods the logs when removing the volume | Major | . | Haohui Mai | Kelvin Chu |
+| [HADOOP-14702](https://issues.apache.org/jira/browse/HADOOP-14702) | Fix formatting issue and regression caused by conversion from APT to Markdown | Minor | documentation | Doris Gu | Doris Gu |
+| [YARN-6872](https://issues.apache.org/jira/browse/YARN-6872) | Ensure apps could run given NodeLabels are disabled post RM switchover/restart | Major | resourcemanager | Sunil G | Sunil G |
+| [HDFS-12217](https://issues.apache.org/jira/browse/HDFS-12217) | HDFS snapshots doesn't capture all open files when one of the open files is deleted | Major | snapshots | Manoj Govindassamy | Manoj Govindassamy |
+| [YARN-6846](https://issues.apache.org/jira/browse/YARN-6846) | Nodemanager can fail to fully delete application local directories when applications are killed | Critical | nodemanager | Jason Lowe | Jason Lowe |
+| [YARN-6678](https://issues.apache.org/jira/browse/YARN-6678) | Handle IllegalStateException in Async Scheduling mode of CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang |
+| [YARN-6932](https://issues.apache.org/jira/browse/YARN-6932) | Fix TestFederationRMFailoverProxyProvider test case failure | Major | . | Arun Suresh | Subru Krishnan |
+| [YARN-6895](https://issues.apache.org/jira/browse/YARN-6895) | [FairScheduler] Preemption reservation may cause regular reservation leaks | Blocker | fairscheduler | Miklos Szegedi | Miklos Szegedi |
+| [HADOOP-13963](https://issues.apache.org/jira/browse/HADOOP-13963) | /bin/bash is hard coded in some of the scripts | Major | scripts | Miklos Szegedi | Ajay Kumar |
+| [HADOOP-14722](https://issues.apache.org/jira/browse/HADOOP-14722) | Azure: BlockBlobInputStream position incorrect after seek | Major | fs/azure | Thomas Marquardt | Thomas Marquardt |
+| [YARN-6951](https://issues.apache.org/jira/browse/YARN-6951) | Fix debug log when Resource Handler chain is enabled | Minor | . | Yang Wang | Yang Wang |
+| [HADOOP-14727](https://issues.apache.org/jira/browse/HADOOP-14727) | Socket not closed properly when reading Configurations with BlockReaderRemote | Blocker | conf | Xiao Chen | Jonathan Eagles |
+| [YARN-6920](https://issues.apache.org/jira/browse/YARN-6920) | Fix resource leak that happens during container re-initialization. | Major | nodemanager | Arun Suresh | Arun Suresh |
+| [HADOOP-14730](https://issues.apache.org/jira/browse/HADOOP-14730) | Support protobuf FileStatus in AdlFileSystem | Major | . | Vishwajeet Dusane | Chris Douglas |
+| [HDFS-12198](https://issues.apache.org/jira/browse/HDFS-12198) | Document missing namenode metrics that were added recently | Minor | documentation | Yiqun Lin | Yiqun Lin |
+| [HADOOP-14680](https://issues.apache.org/jira/browse/HADOOP-14680) | Azure: IndexOutOfBoundsException in BlockBlobInputStream | Minor | fs/azure | Rajesh Balamohan | Thomas Marquardt |
+| [YARN-6757](https://issues.apache.org/jira/browse/YARN-6757) | Refactor the usage of yarn.nodemanager.linux-container-executor.cgroups.mount-path | Minor | nodemanager | Miklos Szegedi | Miklos Szegedi |
+| [MAPREDUCE-6927](https://issues.apache.org/jira/browse/MAPREDUCE-6927) | MR job should only set tracking url if history was successfully written | Major | . | Eric Badger | Eric Badger |
+| [YARN-6890](https://issues.apache.org/jira/browse/YARN-6890) | If UI is not secured, we allow user to kill other users' job even yarn cluster is secured. | Critical | . | Sumana Sathish | Junping Du |
+| [HDFS-10326](https://issues.apache.org/jira/browse/HDFS-10326) | Disable setting tcp socket send/receive buffers for write pipelines | Major | datanode, hdfs | Daryn Sharp | Daryn Sharp |
+| [HADOOP-14598](https://issues.apache.org/jira/browse/HADOOP-14598) | Blacklist Http/HttpsFileSystem in FsUrlStreamHandlerFactory | Major | fs/azure, test | Steve Loughran | Steve Loughran |
+| [YARN-6515](https://issues.apache.org/jira/browse/YARN-6515) | Fix warnings from Spotbugs in hadoop-yarn-server-nodemanager | Major | nodemanager | Naganarasimha G R | Naganarasimha G R |
+| [HDFS-12157](https://issues.apache.org/jira/browse/HDFS-12157) | Do fsyncDirectory(..) outside of FSDataset lock | Critical | datanode | Vinayakumar B | Vinayakumar B |
+| [HDFS-12278](https://issues.apache.org/jira/browse/HDFS-12278) | LeaseManager operations are inefficient in 2.8. | Blocker | namenode | Rushabh S Shah | Rushabh S Shah |
+| [HADOOP-14743](https://issues.apache.org/jira/browse/HADOOP-14743) | CompositeGroupsMapping should not swallow exceptions | Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-14754](https://issues.apache.org/jira/browse/HADOOP-14754) | TestCommonConfigurationFields failed: core-default.xml has 2 wasb properties missing in classes | Minor | common, fs/azure | John Zhuge | John Zhuge |
+| [HADOOP-14760](https://issues.apache.org/jira/browse/HADOOP-14760) | Add missing override to LoadBalancingKMSClientProvider | Minor | kms | Xiao Chen | Xiao Chen |
+| [YARN-5927](https://issues.apache.org/jira/browse/YARN-5927) | BaseContainerManagerTest::waitForNMContainerState timeout accounting is not accurate | Trivial | . | Miklos Szegedi | Kai Sasaki |
+| [YARN-6967](https://issues.apache.org/jira/browse/YARN-6967) | Limit application attempt's diagnostic message size thoroughly | Major | resourcemanager | Chengbing Liu | Chengbing Liu |
+| [HDFS-11303](https://issues.apache.org/jira/browse/HDFS-11303) | Hedged read might hang infinitely if read data from all DN failed | Major | hdfs-client | Chen Zhang | Chen Zhang |
+| [YARN-6996](https://issues.apache.org/jira/browse/YARN-6996) | Change javax.cache library implementation from JSR107 to Apache Geronimo | Blocker | . | Ray Chiang | Ray Chiang |
+| [YARN-6987](https://issues.apache.org/jira/browse/YARN-6987) | Log app attempt during InvalidStateTransition | Major | . | Jonathan Eagles | Jonathan Eagles |
+| [HDFS-11696](https://issues.apache.org/jira/browse/HDFS-11696) | Fix warnings from Spotbugs in hadoop-hdfs | Major | . | Yiqun Lin | Yiqun Lin |
+| [HDFS-12066](https://issues.apache.org/jira/browse/HDFS-12066) | When Namenode is in safemode,may not allowed to remove an user's erasure coding policy | Major | hdfs | lufei | lufei |
+| [HDFS-12054](https://issues.apache.org/jira/browse/HDFS-12054) | FSNamesystem#addErasureCodingPolicies should call checkNameNodeSafeMode() to ensure Namenode is not in safemode | Major | hdfs | lufei | lufei |
+| [YARN-7014](https://issues.apache.org/jira/browse/YARN-7014) | container-executor has off-by-one error which can corrupt the heap | Critical | yarn | Shane Kumpf | Jason Lowe |
+| [HADOOP-14773](https://issues.apache.org/jira/browse/HADOOP-14773) | Extend ZKCuratorManager API for more reusability | Major | . | Íñigo Goiri | Íñigo Goiri |
+| [YARN-6965](https://issues.apache.org/jira/browse/YARN-6965) | Duplicate instantiation in FairSchedulerQueueInfo | Minor | fairscheduler | Masahiro Tanaka | Masahiro Tanaka |
+| [YARN-7020](https://issues.apache.org/jira/browse/YARN-7020) | TestAMRMProxy#testAMRMProxyTokenRenewal is flakey | Major | . | Robert Kanter | Robert Kanter |
+| [MAPREDUCE-6940](https://issues.apache.org/jira/browse/MAPREDUCE-6940) | Copy-paste error in the TaskAttemptUnsuccessfulCompletionEvent constructor | Minor | . | Oleg Danilov | Oleg Danilov |
+| [MAPREDUCE-6936](https://issues.apache.org/jira/browse/MAPREDUCE-6936) | Remove unnecessary dependency of hadoop-yarn-server-common from hadoop-mapreduce-client-common | Major | mrv2 | Haibo Chen | Haibo Chen |
+| [HDFS-12250](https://issues.apache.org/jira/browse/HDFS-12250) | Reduce usage of FsPermissionExtension in unit tests | Minor | test | Chris Douglas | Chris Douglas |
+| [HDFS-12316](https://issues.apache.org/jira/browse/HDFS-12316) | Verify HDFS snapshot deletion doesn't crash the ongoing file writes | Major | . | Manoj Govindassamy | Manoj Govindassamy |
+| [YARN-7007](https://issues.apache.org/jira/browse/YARN-7007) | NPE in RM while using YarnClient.getApplications() | Major | . | Lingfeng Su | Lingfeng Su |
+| [HDFS-12325](https://issues.apache.org/jira/browse/HDFS-12325) | SFTPFileSystem operations should restore cwd | Major | . | Namit Maheshwari | Chen Liang |
+| [HDFS-11738](https://issues.apache.org/jira/browse/HDFS-11738) | Hedged pread takes more time when block moved from initial locations | Major | hdfs-client | Vinayakumar B | Vinayakumar B |
+| [YARN-2416](https://issues.apache.org/jira/browse/YARN-2416) | InvalidStateTransitonException in ResourceManager if AMLauncher does not receive response for startContainers() call in time | Critical | resourcemanager | Jian Fang | Jonathan Eagles |
+| [YARN-7048](https://issues.apache.org/jira/browse/YARN-7048) | Fix tests faking kerberos to explicitly set ugi auth type | Major | yarn | Daryn Sharp | Daryn Sharp |
+| [HADOOP-14687](https://issues.apache.org/jira/browse/HADOOP-14687) | AuthenticatedURL will reuse bad/expired session cookies | Critical | common | Daryn Sharp | Daryn Sharp |
+| [YARN-6251](https://issues.apache.org/jira/browse/YARN-6251) | Do async container release to prevent deadlock during container updates | Major | . | Arun Suresh | Arun Suresh |
+| [YARN-7090](https://issues.apache.org/jira/browse/YARN-7090) | testRMRestartAfterNodeLabelDisabled get failed when CapacityScheduler is configured | Major | test | Yesha Vora | Wangda Tan |
+| [HDFS-12318](https://issues.apache.org/jira/browse/HDFS-12318) | Fix IOException condition for openInfo in DFSInputStream | Major | . | legend | legend |
+| [YARN-7074](https://issues.apache.org/jira/browse/YARN-7074) | Fix NM state store update comment | Minor | nodemanager | Botong Huang | Botong Huang |
+| [HDFS-12344](https://issues.apache.org/jira/browse/HDFS-12344) | LocatedFileStatus regression: no longer accepting null FSPermission | Minor | fs | Ewan Higgs | Ewan Higgs |
+| [YARN-6640](https://issues.apache.org/jira/browse/YARN-6640) | AM heartbeat stuck when responseId overflows MAX\_INT | Blocker | . | Botong Huang | Botong Huang |
+| [HDFS-12319](https://issues.apache.org/jira/browse/HDFS-12319) | DirectoryScanner will throw IllegalStateException when Multiple BP's are present | Blocker | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-12299](https://issues.apache.org/jira/browse/HDFS-12299) | Race Between update pipeline and DN Re-Registration | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [YARN-7052](https://issues.apache.org/jira/browse/YARN-7052) | RM SchedulingMonitor gives no indication why the spawned thread crashed. | Critical | yarn | Eric Payne | Eric Payne |
+| [YARN-7087](https://issues.apache.org/jira/browse/YARN-7087) | NM failed to perform log aggregation due to absent container | Blocker | log-aggregation | Jason Lowe | Jason Lowe |
+| [HDFS-12215](https://issues.apache.org/jira/browse/HDFS-12215) | DataNode#transferBlock does not create its daemon in the xceiver thread group | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HDFS-12248](https://issues.apache.org/jira/browse/HDFS-12248) | SNN will not upload fsimage on IOE and Interrupted exceptions | Critical | ha, namenode, rolling upgrades | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-12358](https://issues.apache.org/jira/browse/HDFS-12358) | Handle IOException when transferring edit log to Journal current dir through JN sync | Major | . | Hanisha Koneru | Hanisha Koneru |
+| [MAPREDUCE-6945](https://issues.apache.org/jira/browse/MAPREDUCE-6945) | TestMapFileOutputFormat missing @after annotation | Minor | . | Ajay Kumar | Ajay Kumar |
+| [YARN-7051](https://issues.apache.org/jira/browse/YARN-7051) | Avoid concurrent modification exception in FifoIntraQueuePreemptionPlugin | Critical | capacity scheduler, scheduler preemption, yarn | Eric Payne | Eric Payne |
+| [YARN-7099](https://issues.apache.org/jira/browse/YARN-7099) | ResourceHandlerModule.parseConfiguredCGroupPath only works for privileged yarn users. | Minor | nodemanager | Miklos Szegedi | Miklos Szegedi |
+| [YARN-7112](https://issues.apache.org/jira/browse/YARN-7112) | TestAMRMProxy is failing with invalid request | Major | . | Jason Lowe | Jason Lowe |
+| [HDFS-12293](https://issues.apache.org/jira/browse/HDFS-12293) | DataNode should log file name on disk error | Major | datanode | Wei-Chiu Chuang | Ajay Kumar |
+| [YARN-7076](https://issues.apache.org/jira/browse/YARN-7076) | yarn application -list -appTypes \<appType\> is not working | Blocker | . | Jian He | Jian He |
+| [YARN-5816](https://issues.apache.org/jira/browse/YARN-5816) | TestDelegationTokenRenewer#testCancelWithMultipleAppSubmissions is still flakey | Minor | resourcemanager, test | Daniel Templeton | Robert Kanter |
+| [YARN-6756](https://issues.apache.org/jira/browse/YARN-6756) | ContainerRequest#executionTypeRequest causes NPE | Critical | . | Jian He | Jian He |
+| [HDFS-12191](https://issues.apache.org/jira/browse/HDFS-12191) | Provide option to not capture the accessTime change of a file to snapshot if no other modification has been done to this file | Major | hdfs, namenode | Yongjun Zhang | Yongjun Zhang |
+| [YARN-6982](https://issues.apache.org/jira/browse/YARN-6982) | Potential issue on setting AMContainerSpec#tokenConf to null before app is completed | Major | . | Rohith Sharma K S | Manikandan R |
+| [YARN-6868](https://issues.apache.org/jira/browse/YARN-6868) | Add test scope to certain entries in hadoop-yarn-server-resourcemanager pom.xml | Major | yarn | Ray Chiang | Ray Chiang |
+| [HDFS-12336](https://issues.apache.org/jira/browse/HDFS-12336) | Listing encryption zones still fails when deleted EZ is not a direct child of snapshottable directory | Minor | encryption, hdfs | Wellington Chevreuil | Wellington Chevreuil |
+| [HADOOP-14814](https://issues.apache.org/jira/browse/HADOOP-14814) | Fix incompatible API change on FsServerDefaults to HADOOP-14104 | Blocker | . | Junping Du | Junping Du |
+| [MAPREDUCE-6931](https://issues.apache.org/jira/browse/MAPREDUCE-6931) | Remove TestDFSIO "Total Throughput" calculation | Critical | benchmarks, test | Dennis Huo | Dennis Huo |
+| [YARN-7115](https://issues.apache.org/jira/browse/YARN-7115) | Move BoundedAppender to org.hadoop.yarn.util pacakge | Major | . | Jian He | Jian He |
+| [YARN-7077](https://issues.apache.org/jira/browse/YARN-7077) | TestAMSimulator and TestNMSimulator fail | Major | test | Akira Ajisaka | Akira Ajisaka |
+| [YARN-7116](https://issues.apache.org/jira/browse/YARN-7116) | CapacityScheduler Web UI: Queue's AM usage is always show on per-user's AM usage. | Major | capacity scheduler, webapp | Wangda Tan | Wangda Tan |
+| [YARN-6726](https://issues.apache.org/jira/browse/YARN-6726) | Fix issues with docker commands executed by container-executor | Major | nodemanager | Shane Kumpf | Shane Kumpf |
+| [HADOOP-14364](https://issues.apache.org/jira/browse/HADOOP-14364) | refresh changelog/release notes with newer Apache Yetus build | Major | build, documentation | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-12317](https://issues.apache.org/jira/browse/HDFS-12317) | HDFS metrics render error in the page of Github | Minor | documentation, metrics | Yiqun Lin | Yiqun Lin |
+| [HADOOP-14824](https://issues.apache.org/jira/browse/HADOOP-14824) | Update ADLS SDK to 2.2.2 for MSI fix | Major | fs/adl | Atul Sikaria | Atul Sikaria |
+| [HDFS-12363](https://issues.apache.org/jira/browse/HDFS-12363) | Possible NPE in BlockManager$StorageInfoDefragmenter#scanAndCompactStorages | Major | namenode | Xiao Chen | Xiao Chen |
+| [YARN-7141](https://issues.apache.org/jira/browse/YARN-7141) | Move logging APIs to slf4j in timelineservice after ATSv2 merge | Minor | . | Varun Saxena | Varun Saxena |
+| [HDFS-11964](https://issues.apache.org/jira/browse/HDFS-11964) | Decoding inputs should be correctly prepared in pread | Major | erasure-coding | Lei (Eddy) Xu | Takanobu Asanuma |
+| [YARN-7120](https://issues.apache.org/jira/browse/YARN-7120) | CapacitySchedulerPage NPE in "Aggregate scheduler counts" section | Minor | . | Eric Payne | Eric Payne |
+| [HADOOP-14674](https://issues.apache.org/jira/browse/HADOOP-14674) | Correct javadoc for getRandomizedTempPath | Major | common | Mukul Kumar Singh | Mukul Kumar Singh |
+| [YARN-7023](https://issues.apache.org/jira/browse/YARN-7023) | Incorrect ReservationId.compareTo() implementation | Minor | reservation system | Oleg Danilov | Oleg Danilov |
+| [HDFS-12383](https://issues.apache.org/jira/browse/HDFS-12383) | Re-encryption updater should handle canceled tasks better | Major | encryption | Xiao Chen | Xiao Chen |
+| [YARN-7152](https://issues.apache.org/jira/browse/YARN-7152) | [ATSv2] Registering timeline client before AMRMClient service init throw exception. | Major | timelineclient | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-12388](https://issues.apache.org/jira/browse/HDFS-12388) | A bad error message in DFSStripedOutputStream | Major | erasure-coding | Kai Zheng | Huafeng Wang |
+| [HADOOP-14820](https://issues.apache.org/jira/browse/HADOOP-14820) | Wasb mkdirs security checks inconsistent with HDFS | Major | fs/azure | Sivaguru Sankaridurg | Sivaguru Sankaridurg |
+| [HDFS-12359](https://issues.apache.org/jira/browse/HDFS-12359) | Re-encryption should operate with minimum KMS ACL requirements. | Major | encryption | Xiao Chen | Xiao Chen |
+| [HDFS-11882](https://issues.apache.org/jira/browse/HDFS-11882) | Precisely calculate acked length of striped block groups in updatePipeline | Critical | erasure-coding, test | Akira Ajisaka | Andrew Wang |
+| [HDFS-12392](https://issues.apache.org/jira/browse/HDFS-12392) | Writing striped file failed due to different cell size | Major | erasure-coding | SammiChen | SammiChen |
+| [YARN-7164](https://issues.apache.org/jira/browse/YARN-7164) | TestAMRMClientOnRMRestart fails sporadically with bind address in use | Major | test | Jason Lowe | Jason Lowe |
+| [YARN-6992](https://issues.apache.org/jira/browse/YARN-6992) | Kill application button is visible even if the application is FINISHED in RM UI | Major | . | Sumana Sathish | Suma Shivaprasad |
+| [HDFS-12357](https://issues.apache.org/jira/browse/HDFS-12357) | Let NameNode to bypass external attribute provider for special user | Major | . | Yongjun Zhang | Yongjun Zhang |
+| [HDFS-12369](https://issues.apache.org/jira/browse/HDFS-12369) | Edit log corruption due to hard lease recovery of not-closed file which has snapshots | Major | namenode | Xiao Chen | Xiao Chen |
+| [HDFS-12404](https://issues.apache.org/jira/browse/HDFS-12404) | Rename hdfs config authorization.provider.bypass.users to attributes.provider.bypass.users | Major | hdfs | Yongjun Zhang | Manoj Govindassamy |
+| [HDFS-12400](https://issues.apache.org/jira/browse/HDFS-12400) | Provide a way for NN to drain the local key cache before re-encryption | Major | encryption | Xiao Chen | Xiao Chen |
+| [YARN-7140](https://issues.apache.org/jira/browse/YARN-7140) | CollectorInfo should have Public visibility | Minor | . | Varun Saxena | Varun Saxena |
+| [YARN-7130](https://issues.apache.org/jira/browse/YARN-7130) | ATSv2 documentation changes post merge | Major | timelineserver | Varun Saxena | Varun Saxena |
+| [HADOOP-14851](https://issues.apache.org/jira/browse/HADOOP-14851) | LambdaTestUtils.eventually() doesn't spin on Assertion failures | Major | test | Steve Loughran | Steve Loughran |
+| [YARN-7181](https://issues.apache.org/jira/browse/YARN-7181) | CPUTimeTracker.updateElapsedJiffies can report negative usage | Major | . | Miklos Szegedi | Miklos Szegedi |
+| [HDFS-12417](https://issues.apache.org/jira/browse/HDFS-12417) | Disable flaky TestDFSStripedOutputStreamWithFailure | Major | test | Chris Douglas | Andrew Wang |
+| [HADOOP-14856](https://issues.apache.org/jira/browse/HADOOP-14856) | Fix AWS, Jetty, HBase, Ehcache entries for NOTICE.txt | Major | . | Ray Chiang | Ray Chiang |
+| [HDFS-12407](https://issues.apache.org/jira/browse/HDFS-12407) | Journal nodes fails to shutdown cleanly if JournalNodeHttpServer or JournalNodeRpcServer fails to start | Major | . | Ajay Kumar | Ajay Kumar |
+| [YARN-7185](https://issues.apache.org/jira/browse/YARN-7185) | ContainerScheduler should only look at availableResource for GUARANTEED containers when OPPORTUNISTIC container queuing is enabled. | Blocker | yarn | Sumana Sathish | Tan, Wangda |
+| [HDFS-12222](https://issues.apache.org/jira/browse/HDFS-12222) | Document and test BlockLocation for erasure-coded files | Major | . | Andrew Wang | Huafeng Wang |
+| [HADOOP-14867](https://issues.apache.org/jira/browse/HADOOP-14867) | Update HDFS Federation setup document, for incorrect property name for secondary name node http address | Major | . | Bharat Viswanadham | Bharat Viswanadham |
+| [YARN-4727](https://issues.apache.org/jira/browse/YARN-4727) | Unable to override the $HADOOP\_CONF\_DIR env variable for container | Major | nodemanager | Terence Yim | Jason Lowe |
+| [MAPREDUCE-6957](https://issues.apache.org/jira/browse/MAPREDUCE-6957) | shuffle hangs after a node manager connection timeout | Major | mrv2 | Jooseong Kim | Jooseong Kim |
+| [YARN-7146](https://issues.apache.org/jira/browse/YARN-7146) | Many RM unit tests failing with FairScheduler | Major | test | Robert Kanter | Robert Kanter |
+| [HDFS-12457](https://issues.apache.org/jira/browse/HDFS-12457) | Revert HDFS-11156 Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API | Major | webhdfs | Andrew Wang | Andrew Wang |
+| [HDFS-12378](https://issues.apache.org/jira/browse/HDFS-12378) | TestClientProtocolForPipelineRecovery#testZeroByteBlockRecovery fails on trunk | Blocker | test | Xiao Chen | Lei (Eddy) Xu |
+| [HDFS-12456](https://issues.apache.org/jira/browse/HDFS-12456) | TestNamenodeMetrics.testSyncAndBlockReportMetric fails | Minor | hdfs, metrics | Bharat Viswanadham | Bharat Viswanadham |
+| [YARN-7163](https://issues.apache.org/jira/browse/YARN-7163) | RMContext need not to be injected to webapp and other Always Running services. | Blocker | resourcemanager | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-12424](https://issues.apache.org/jira/browse/HDFS-12424) | Datatable sorting on the Datanode Information page in the Namenode UI is broken | Major | . | Shawna Martell | Shawna Martell |
+| [HADOOP-14853](https://issues.apache.org/jira/browse/HADOOP-14853) | hadoop-mapreduce-client-app is not a client module | Major | . | Haibo Chen | Haibo Chen |
+| [HDFS-12323](https://issues.apache.org/jira/browse/HDFS-12323) | NameNode terminates after full GC thinking QJM unresponsive if full GC is much longer than timeout | Major | namenode, qjm | Erik Krogen | Erik Krogen |
+| [HDFS-10701](https://issues.apache.org/jira/browse/HDFS-10701) | TestDFSStripedOutputStreamWithFailure#testBlockTokenExpired occasionally fails | Major | erasure-coding | Wei-Chiu Chuang | SammiChen |
+| [YARN-6977](https://issues.apache.org/jira/browse/YARN-6977) | Node information is not provided for non am containers in RM logs | Major | capacity scheduler | Sumana Sathish | Suma Shivaprasad |
+| [YARN-7149](https://issues.apache.org/jira/browse/YARN-7149) | Cross-queue preemption sometimes starves an underserved queue | Major | capacity scheduler | Eric Payne | Eric Payne |
+| [YARN-7192](https://issues.apache.org/jira/browse/YARN-7192) | Add a pluggable StateMachine Listener that is notified of NM Container State changes | Major | . | Arun Suresh | Arun Suresh |
+| [HADOOP-14835](https://issues.apache.org/jira/browse/HADOOP-14835) | mvn site build throws SAX errors | Blocker | build, site | Allen Wittenauer | Andrew Wang |
+| [MAPREDUCE-6960](https://issues.apache.org/jira/browse/MAPREDUCE-6960) | Shuffle Handler prints disk error stack traces for every read failure. | Major | . | Kuhu Shukla | Kuhu Shukla |
+| [HDFS-12480](https://issues.apache.org/jira/browse/HDFS-12480) | TestNameNodeMetrics#testTransactionAndCheckpointMetrics Fails in trunk | Blocker | test | Brahma Reddy Battula | Hanisha Koneru |
+| [HDFS-11799](https://issues.apache.org/jira/browse/HDFS-11799) | Introduce a config to allow setting up write pipeline with fewer nodes than replication factor | Major | . | Yongjun Zhang | Brahma Reddy Battula |
+| [HDFS-12449](https://issues.apache.org/jira/browse/HDFS-12449) | TestReconstructStripedFile.testNNSendsErasureCodingTasks randomly cannot finish in 60s | Major | erasure-coding | SammiChen | SammiChen |
+| [HDFS-12437](https://issues.apache.org/jira/browse/HDFS-12437) | Fix test setup in TestLeaseRecoveryStriped | Major | erasure-coding, test | Arpit Agarwal | Andrew Wang |
+| [YARN-7196](https://issues.apache.org/jira/browse/YARN-7196) | Fix finicky TestContainerManager tests | Major | . | Arun Suresh | Arun Suresh |
+| [YARN-6771](https://issues.apache.org/jira/browse/YARN-6771) | Use classloader inside configuration class to make new classes | Major | . | Jongyoul Lee | Jongyoul Lee |
+| [HDFS-12526](https://issues.apache.org/jira/browse/HDFS-12526) | FSDirectory should use Time.monotonicNow for durations | Minor | . | Chetna Chaudhari | Bharat Viswanadham |
+| [YARN-6968](https://issues.apache.org/jira/browse/YARN-6968) | Hardcoded absolute pathname in DockerLinuxContainerRuntime | Major | nodemanager | Miklos Szegedi | Eric Badger |
+| [HDFS-12371](https://issues.apache.org/jira/browse/HDFS-12371) | "BlockVerificationFailures" and "BlocksVerified" show up as 0 in Datanode JMX | Major | metrics | Sai Nukavarapu | Hanisha Koneru |
+| [MAPREDUCE-6964](https://issues.apache.org/jira/browse/MAPREDUCE-6964) | BaileyBorweinPlouffe should use Time.monotonicNow for measuring durations | Minor | examples | Chetna Chaudhari | Chetna Chaudhari |
+| [YARN-6991](https://issues.apache.org/jira/browse/YARN-6991) | "Kill application" button does not show error if other user tries to kill the application for secure cluster | Major | . | Sumana Sathish | Suma Shivaprasad |
+| [YARN-7034](https://issues.apache.org/jira/browse/YARN-7034) | DefaultLinuxContainerRuntime and DockerLinuxContainerRuntime sends client environment variables to container-executor | Blocker | nodemanager | Miklos Szegedi | Miklos Szegedi |
+| [HDFS-12523](https://issues.apache.org/jira/browse/HDFS-12523) | Thread pools in ErasureCodingWorker do not shutdown | Major | erasure-coding | Lei (Eddy) Xu | Huafeng Wang |
+| [MAPREDUCE-6966](https://issues.apache.org/jira/browse/MAPREDUCE-6966) | DistSum should use Time.monotonicNow for measuring durations | Minor | . | Chetna Chaudhari | Chetna Chaudhari |
+| [MAPREDUCE-6965](https://issues.apache.org/jira/browse/MAPREDUCE-6965) | QuasiMonteCarlo should use Time.monotonicNow for measuring durations | Minor | examples | Chetna Chaudhari | Chetna Chaudhari |
+| [MAPREDUCE-6967](https://issues.apache.org/jira/browse/MAPREDUCE-6967) | gridmix/SleepReducer should use Time.monotonicNow for measuring durations | Minor | . | Chetna Chaudhari | Chetna Chaudhari |
+| [YARN-7153](https://issues.apache.org/jira/browse/YARN-7153) | Remove duplicated code in AMRMClientAsyncImpl.java | Minor | client | Sen Zhao | Sen Zhao |
+| [HADOOP-14897](https://issues.apache.org/jira/browse/HADOOP-14897) | Loosen compatibility guidelines for native dependencies | Blocker | documentation, native | Chris Douglas | Daniel Templeton |
+| [HADOOP-14903](https://issues.apache.org/jira/browse/HADOOP-14903) | Add json-smart explicitly to pom.xml | Major | common | Ray Chiang | Ray Chiang |
+| [YARN-7118](https://issues.apache.org/jira/browse/YARN-7118) | AHS REST API can return NullPointerException | Major | . | Prabhu Joseph | Billie Rinaldi |
+| [HDFS-12495](https://issues.apache.org/jira/browse/HDFS-12495) | TestPendingInvalidateBlock#testPendingDeleteUnknownBlocks fails intermittently | Major | . | Eric Badger | Eric Badger |
+| [MAPREDUCE-6968](https://issues.apache.org/jira/browse/MAPREDUCE-6968) | Staging directory erasure coding config property has a typo | Major | client | Jason Lowe | Jason Lowe |
+| [HADOOP-14822](https://issues.apache.org/jira/browse/HADOOP-14822) | hadoop-project/pom.xml is executable | Minor | . | Akira Ajisaka | Ajay Kumar |
+| [YARN-7157](https://issues.apache.org/jira/browse/YARN-7157) | Add admin configuration to filter per-user's apps in secure cluster | Major | webapp | Sunil G | Sunil G |
+| [YARN-7253](https://issues.apache.org/jira/browse/YARN-7253) | Shared Cache Manager daemon command listed as admin subcmd in yarn script | Trivial | . | Chris Trezzo | Chris Trezzo |
+| [YARN-7257](https://issues.apache.org/jira/browse/YARN-7257) | AggregatedLogsBlock reports a bad 'end' value as a bad 'start' value | Major | log-aggregation | Jason Lowe | Jason Lowe |
+| [HDFS-12458](https://issues.apache.org/jira/browse/HDFS-12458) | TestReencryptionWithKMS fails regularly | Major | encryption, test | Konstantin Shvachko | Xiao Chen |
+
+
+### TESTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-5548](https://issues.apache.org/jira/browse/YARN-5548) | Use MockRMMemoryStateStore to reduce test failures | Major | . | Bibin A Chundatt | Bibin A Chundatt |
+| [HDFS-2319](https://issues.apache.org/jira/browse/HDFS-2319) | Add test cases for FSshell -stat | Trivial | test | XieXianshan | Bharat Viswanadham |
+| [HADOOP-14245](https://issues.apache.org/jira/browse/HADOOP-14245) | Use Mockito.when instead of Mockito.stub | Minor | test | Akira Ajisaka | Andras Bokor |
+| [YARN-5349](https://issues.apache.org/jira/browse/YARN-5349) | TestWorkPreservingRMRestart#testUAMRecoveryOnRMWorkPreservingRestart fail intermittently | Minor | . | sandflee | Jason Lowe |
+| [HDFS-11988](https://issues.apache.org/jira/browse/HDFS-11988) | Verify HDFS Snapshots with open files captured are safe across truncates and appends on current version file | Major | hdfs, snapshots | Manoj Govindassamy | Manoj Govindassamy |
+| [HADOOP-14729](https://issues.apache.org/jira/browse/HADOOP-14729) | Upgrade JUnit 3 test cases to JUnit 4 | Major | . | Akira Ajisaka | Ajay Kumar |
+| [HDFS-11912](https://issues.apache.org/jira/browse/HDFS-11912) | Add a snapshot unit test with randomized file IO operations | Minor | hdfs | George Huang | George Huang |
+| [MAPREDUCE-6953](https://issues.apache.org/jira/browse/MAPREDUCE-6953) | Skip the testcase testJobWithChangePriority if FairScheduler is used | Major | client | Peter Bacsko | Peter Bacsko |
+
+
+### SUB-TASKS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-5300](https://issues.apache.org/jira/browse/YARN-5300) | Exclude generated federation protobuf sources from YARN Javadoc/findbugs build | Minor | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5156](https://issues.apache.org/jira/browse/YARN-5156) | YARN\_CONTAINER\_FINISHED of YARN\_CONTAINERs will always have running state | Major | timelineserver | Li Lu | Vrushali C |
+| [YARN-3662](https://issues.apache.org/jira/browse/YARN-3662) | Federation Membership State Store internal APIs | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5229](https://issues.apache.org/jira/browse/YARN-5229) | Refactor #isApplicationEntity and #getApplicationEvent from HBaseTimelineWriterImpl | Minor | timelineserver | Joep Rottinghuis | Vrushali C |
+| [YARN-5406](https://issues.apache.org/jira/browse/YARN-5406) | In-memory based implementation of the FederationMembershipStateStore | Major | nodemanager, resourcemanager | Subru Krishnan | Ellen Hui |
+| [YARN-5390](https://issues.apache.org/jira/browse/YARN-5390) | Federation Subcluster Resolver | Major | nodemanager, resourcemanager | Carlo Curino | Ellen Hui |
+| [YARN-5307](https://issues.apache.org/jira/browse/YARN-5307) | Federation Application State Store internal APIs | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-3664](https://issues.apache.org/jira/browse/YARN-3664) | Federation PolicyStore internal APIs | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5408](https://issues.apache.org/jira/browse/YARN-5408) | Compose Federation membership/application/policy APIs into an uber FederationStateStore API | Major | nodemanager, resourcemanager | Subru Krishnan | Ellen Hui |
+| [YARN-5407](https://issues.apache.org/jira/browse/YARN-5407) | In-memory based implementation of the FederationApplicationStateStore, FederationPolicyStateStore | Major | nodemanager, resourcemanager | Subru Krishnan | Ellen Hui |
+| [YARN-5519](https://issues.apache.org/jira/browse/YARN-5519) | Add SubClusterId in AddApplicationHomeSubClusterResponse for Router Failover | Major | nodemanager, resourcemanager | Giovanni Matteo Fumarola | Ellen Hui |
+| [YARN-3672](https://issues.apache.org/jira/browse/YARN-3672) | Create Facade for Federation State and Policy Store | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5467](https://issues.apache.org/jira/browse/YARN-5467) | InputValidator for the FederationStateStore internal APIs | Major | nodemanager, resourcemanager | Subru Krishnan | Giovanni Matteo Fumarola |
+| [YARN-3673](https://issues.apache.org/jira/browse/YARN-3673) | Create a FailoverProxy for Federation services | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-3671](https://issues.apache.org/jira/browse/YARN-3671) | Integrate Federation services with ResourceManager | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5612](https://issues.apache.org/jira/browse/YARN-5612) | Return SubClusterId in FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover | Major | nodemanager, resourcemanager | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-5601](https://issues.apache.org/jira/browse/YARN-5601) | Make the RM epoch base value configurable | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5323](https://issues.apache.org/jira/browse/YARN-5323) | Policies APIs (for Router and AMRMProxy policies) | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5324](https://issues.apache.org/jira/browse/YARN-5324) | Stateless Federation router policies implementation | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5325](https://issues.apache.org/jira/browse/YARN-5325) | Stateless ARMRMProxy policies implementation | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5638](https://issues.apache.org/jira/browse/YARN-5638) | Introduce a collector timestamp to uniquely identify collectors creation order in collector discovery | Major | timelineserver | Li Lu | Li Lu |
+| [YARN-3649](https://issues.apache.org/jira/browse/YARN-3649) | Allow configurable prefix for hbase table names (like prod, exp, test etc) | Major | timelineserver | Vrushali C | Vrushali C |
+| [YARN-5715](https://issues.apache.org/jira/browse/YARN-5715) | introduce entity prefix for return and sort order | Critical | timelineserver | Sangjin Lee | Rohith Sharma K S |
+| [YARN-5391](https://issues.apache.org/jira/browse/YARN-5391) | PolicyManager to tie together Router/AMRM Federation policies | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5265](https://issues.apache.org/jira/browse/YARN-5265) | Make HBase configuration for the timeline service configurable | Major | timelineserver | Joep Rottinghuis | Joep Rottinghuis |
+| [YARN-3359](https://issues.apache.org/jira/browse/YARN-3359) | Recover collector list when RM fails over | Major | resourcemanager | Junping Du | Li Lu |
+| [YARN-5634](https://issues.apache.org/jira/browse/YARN-5634) | Simplify initialization/use of RouterPolicy via a RouterPolicyFacade | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5792](https://issues.apache.org/jira/browse/YARN-5792) | adopt the id prefix for YARN, MR, and DS entities | Major | timelineserver | Sangjin Lee | Varun Saxena |
+| [YARN-5676](https://issues.apache.org/jira/browse/YARN-5676) | Add a HashBasedRouterPolicy, and small policies and test refactoring. | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5872](https://issues.apache.org/jira/browse/YARN-5872) | Add AlwayReject policies for router and amrmproxy. | Major | nodemanager, resourcemanager | Carlo Curino | Carlo Curino |
+| [YARN-5905](https://issues.apache.org/jira/browse/YARN-5905) | Update the RM webapp host that is reported as part of Federation membership to current primary RM's IP | Minor | federation, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-5739](https://issues.apache.org/jira/browse/YARN-5739) | Provide timeline reader API to list available timeline entity types for one application | Major | timelinereader | Li Lu | Li Lu |
+| [MAPREDUCE-6818](https://issues.apache.org/jira/browse/MAPREDUCE-6818) | Remove direct reference to TimelineClientImpl | Major | . | Li Lu | Li Lu |
+| [YARN-5585](https://issues.apache.org/jira/browse/YARN-5585) | [Atsv2] Reader side changes for entity prefix and support for pagination via additional filters | Critical | timelinereader | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-5980](https://issues.apache.org/jira/browse/YARN-5980) | Update documentation for single node hbase deploy | Major | timelineserver | Vrushali C | Vrushali C |
+| [YARN-5378](https://issues.apache.org/jira/browse/YARN-5378) | Accommodate app-id-\>cluster mapping | Major | timelineserver | Joep Rottinghuis | Sangjin Lee |
+| [YARN-6064](https://issues.apache.org/jira/browse/YARN-6064) | Support fromId for flowRuns and flow/flowRun apps REST API's | Major | timelinereader | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-6094](https://issues.apache.org/jira/browse/YARN-6094) | Update the coprocessor to be a dynamically loaded one | Major | timelineserver | Vrushali C | Vrushali C |
+| [YARN-5410](https://issues.apache.org/jira/browse/YARN-5410) | Bootstrap Router server module | Major | nodemanager, resourcemanager | Subru Krishnan | Giovanni Matteo Fumarola |
+| [YARN-6253](https://issues.apache.org/jira/browse/YARN-6253) | FlowAcitivityColumnPrefix.store(byte[] rowKey, ...) drops timestamp | Major | . | Haibo Chen | Haibo Chen |
+| [YARN-6190](https://issues.apache.org/jira/browse/YARN-6190) | Validation and synchronization fixes in LocalityMulticastAMRMProxyPolicy | Minor | federation | Botong Huang | Botong Huang |
+| [YARN-6027](https://issues.apache.org/jira/browse/YARN-6027) | Support fromid(offset) filter for /flows API | Major | timelineserver | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-6247](https://issues.apache.org/jira/browse/YARN-6247) | Share a single instance of SubClusterResolver instead of instantiating one per AM | Minor | . | Botong Huang | Botong Huang |
+| [YARN-6256](https://issues.apache.org/jira/browse/YARN-6256) | Add FROM\_ID info key for timeline entities in reader response. | Major | timelineserver | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-6237](https://issues.apache.org/jira/browse/YARN-6237) | Move UID constant to TimelineReaderUtils | Major | timelinereader | Rohith Sharma K S | Rohith Sharma K S |
+| [YARN-6318](https://issues.apache.org/jira/browse/YARN-6318) | timeline service schema creator fails if executed from a remote machine | Minor | timelineserver | Sangjin Lee | Sangjin Lee |
+| [YARN-6146](https://issues.apache.org/jira/browse/YARN-6146) | Add Builder methods for TimelineEntityFilters | Major | timelineserver | Rohith Sharma K S | Haibo Chen |
+| [YARN-5602](https://issues.apache.org/jira/browse/YARN-5602) | Utils for Federation State and Policy Store | Major | nodemanager, resourcemanager | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-6203](https://issues.apache.org/jira/browse/YARN-6203) | Occasional test failure in TestWeightedRandomRouterPolicy | Minor | federation | Botong Huang | Carlo Curino |
+| [YARN-3663](https://issues.apache.org/jira/browse/YARN-3663) | Federation State and Policy Store (DBMS implementation) | Major | nodemanager, resourcemanager | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-5411](https://issues.apache.org/jira/browse/YARN-5411) | Create a proxy chain for ApplicationClientProtocol in the Router | Major | nodemanager, resourcemanager | Subru Krishnan | Giovanni Matteo Fumarola |
+| [YARN-5413](https://issues.apache.org/jira/browse/YARN-5413) | Create a proxy chain for ResourceManager Admin API in the Router | Major | nodemanager, resourcemanager | Subru Krishnan | Giovanni Matteo Fumarola |
+| [YARN-5531](https://issues.apache.org/jira/browse/YARN-5531) | UnmanagedAM pool manager for federating application across clusters | Major | nodemanager, resourcemanager | Subru Krishnan | Botong Huang |
+| [YARN-6666](https://issues.apache.org/jira/browse/YARN-6666) | Fix unit test failure in TestRouterClientRMService | Minor | . | Botong Huang | Botong Huang |
+| [YARN-6484](https://issues.apache.org/jira/browse/YARN-6484) | [Documentation] Documenting the YARN Federation feature | Major | nodemanager, resourcemanager | Subru Krishnan | Carlo Curino |
+| [YARN-6658](https://issues.apache.org/jira/browse/YARN-6658) | Remove columnFor() methods of Columns in HBaseTimeline backend | Major | . | Haibo Chen | Haibo Chen |
+| [YARN-3666](https://issues.apache.org/jira/browse/YARN-3666) | Federation Intercepting and propagating AM- home RM communications | Major | nodemanager, resourcemanager | Kishore Chaliparambil | Botong Huang |
+| [YARN-5647](https://issues.apache.org/jira/browse/YARN-5647) | [ATSv2 Security] Collector side changes for loading auth filters and principals | Major | timelineserver | Varun Saxena | Varun Saxena |
+| [YARN-6511](https://issues.apache.org/jira/browse/YARN-6511) | Federation: transparently spanning application across multiple sub-clusters | Major | . | Botong Huang | Botong Huang |
+| [YARN-6638](https://issues.apache.org/jira/browse/YARN-6638) | [ATSv2 Security] Timeline reader side changes for loading auth filters and principals | Major | timelineserver | Varun Saxena | Varun Saxena |
+| [YARN-6724](https://issues.apache.org/jira/browse/YARN-6724) | Add ability to blacklist sub-clusters when invoking Routing policies | Major | router | Subru Krishnan | Giovanni Matteo Fumarola |
+| [YARN-5648](https://issues.apache.org/jira/browse/YARN-5648) | [ATSv2 Security] Client side changes for authentication | Major | timelineserver | Varun Saxena | Varun Saxena |
+| [YARN-3659](https://issues.apache.org/jira/browse/YARN-3659) | Federation: routing client invocations transparently to multiple RMs | Major | client, resourcemanager, router | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola |
+| [YARN-5067](https://issues.apache.org/jira/browse/YARN-5067) | Support specifying resources for AM containers in SLS | Major | scheduler-load-simulator | Wangda Tan | Yufei Gu |
+| [YARN-6681](https://issues.apache.org/jira/browse/YARN-6681) | Eliminate double-copy of child queues in canAssignToThisQueue | Major | resourcemanager | Daryn Sharp | Daryn Sharp |
+| [HADOOP-14615](https://issues.apache.org/jira/browse/HADOOP-14615) | Add ServiceOperations.stopQuietly that accept slf4j logger API | Major | . | Wenxin He | Wenxin He |
+| [HADOOP-14617](https://issues.apache.org/jira/browse/HADOOP-14617) | Add ReflectionUtils.logThreadInfo that accept slf4j logger API | Major | . | Wenxin He | Wenxin He |
+| [HADOOP-14571](https://issues.apache.org/jira/browse/HADOOP-14571) | Deprecate public APIs relate to log4j1 | Major | . | Akira Ajisaka | Wenxin He |
+| [HADOOP-14587](https://issues.apache.org/jira/browse/HADOOP-14587) | Use GenericTestUtils.setLogLevel when available in hadoop-common | Major | . | Wenxin He | Wenxin He |
+| [YARN-6776](https://issues.apache.org/jira/browse/YARN-6776) | Refactor ApplicaitonMasterService to move actual processing logic to a separate class | Minor | . | Arun Suresh | Arun Suresh |
+| [HADOOP-14638](https://issues.apache.org/jira/browse/HADOOP-14638) | Replace commons-logging APIs with slf4j in StreamPumper | Major | . | Wenxin He | Wenxin He |
+| [YARN-6801](https://issues.apache.org/jira/browse/YARN-6801) | NPE in RM while setting collectors map in NodeHeartbeatResponse | Major | timelineserver | Vrushali C | Vrushali C |
+| [YARN-6807](https://issues.apache.org/jira/browse/YARN-6807) | Adding required missing configs to Federation configuration guide based on e2e testing | Major | documentation, federation | Subru Krishnan | Tanuj Nayak |
+| [YARN-6815](https://issues.apache.org/jira/browse/YARN-6815) | [Bug] FederationStateStoreFacade return behavior should be consistent irrespective of whether caching is enabled or not | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-6821](https://issues.apache.org/jira/browse/YARN-6821) | Move FederationStateStore SQL DDL files from test resource to sbin | Major | nodemanager, resourcemanager | Subru Krishnan | Subru Krishnan |
+| [YARN-6706](https://issues.apache.org/jira/browse/YARN-6706) | Refactor ContainerScheduler to make oversubscription change easier | Major | . | Haibo Chen | Haibo Chen |
+| [HADOOP-14642](https://issues.apache.org/jira/browse/HADOOP-14642) | wasb: add support for caching Authorization and SASKeys | Major | fs/azure | Sivaguru Sankaridurg | Sivaguru Sankaridurg |
+| [YARN-6777](https://issues.apache.org/jira/browse/YARN-6777) | Support for ApplicationMasterService processing chain of interceptors | Major | . | Arun Suresh | Arun Suresh |
+| [YARN-6775](https://issues.apache.org/jira/browse/YARN-6775) | CapacityScheduler: Improvements to assignContainers, avoid unnecessary canAssignToUser/Queue calls | Major | capacityscheduler | Nathan Roberts | Nathan Roberts |
+| [YARN-4455](https://issues.apache.org/jira/browse/YARN-4455) | Support fetching metrics by time range | Major | timelineserver | Varun Saxena | Varun Saxena |
+| [YARN-6685](https://issues.apache.org/jira/browse/YARN-6685) | Add job count in to SLS JSON input format | Major | scheduler-load-simulator | Yufei Gu | Yufei Gu |
+| [YARN-6850](https://issues.apache.org/jira/browse/YARN-6850) | Ensure that supplemented timestamp is stored only for flow run metrics | Major | timelineserver | Vrushali C | Varun Saxena |
+| [YARN-6733](https://issues.apache.org/jira/browse/YARN-6733) | Add table for storing sub-application entities | Major | timelineserver | Vrushali C | Vrushali C |
+| [HADOOP-14518](https://issues.apache.org/jira/browse/HADOOP-14518) | Customize User-Agent header sent in HTTP/HTTPS requests by WASB. | Minor | fs/azure | Georgi Chalakov | Georgi Chalakov |
+| [YARN-6804](https://issues.apache.org/jira/browse/YARN-6804) | Allow custom hostname for docker containers in native services | Major | yarn-native-services | Billie Rinaldi | Billie Rinaldi |
+| [YARN-6866](https://issues.apache.org/jira/browse
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-2960. Add documentation for
the YARN shared cache.
Posted by jh...@apache.org.
YARN-2960. Add documentation for the YARN shared cache.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e76f85b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e76f85b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e76f85b
Branch: refs/heads/YARN-5734
Commit: 7e76f85bc68166b01b51fcf6ba4b3fd9281d4a03
Parents: b733348
Author: Chris Trezzo <ct...@apache.org>
Authored: Thu Oct 5 10:38:41 2017 -0700
Committer: Chris Trezzo <ct...@apache.org>
Committed: Thu Oct 5 10:38:41 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/site.xml | 1 +
.../src/site/markdown/SharedCache.md | 168 +++++++++++++++++++
2 files changed, 169 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e76f85b/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index a88f0e3..44551fe 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -142,6 +142,7 @@
<item name="Opportunistic Containers" href="hadoop-yarn/hadoop-yarn-site/OpportunisticContainers.html"/>
<item name="YARN Federation" href="hadoop-yarn/hadoop-yarn-site/Federation.html"/>
<item name="Resource Profiles" href="hadoop-yarn/hadoop-yarn-site/ResourceProfiles.html"/>
+ <item name="Shared Cache" href="hadoop-yarn/hadoop-yarn-site/SharedCache.html"/>
</menu>
<menu name="YARN REST APIs" inherit="top">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e76f85b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SharedCache.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SharedCache.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SharedCache.md
new file mode 100644
index 0000000..ea50e91
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/SharedCache.md
@@ -0,0 +1,168 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+YARN Shared Cache
+===================
+
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
+
+Overview
+--------
+
+The YARN Shared Cache provides the facility to upload and manage shared
+application resources to HDFS in a safe and scalable manner. YARN applications
+can leverage resources uploaded by other applications or previous
+runs of the same application without having to reupload and localize identical files
+multiple times. This will save network resources and reduce YARN application
+startup time.
+
+Current Status and Future Plans
+------------
+
+Currently the YARN Shared Cache is released and ready to use. The major
+components are implemented and have been deployed in a large-scale
+production setting. There are still some pieces missing (i.e. strong
+authentication). These missing features will be implemented as part of a
+follow-up phase 2 effort. Please see
+[YARN-7282](https://issues.apache.org/jira/browse/YARN-7282) for more information.
+
+Architecture
+------------
+
+The shared cache feature consists of 4 major components:
+
+1. The shared cache client.
+2. The HDFS directory that acts as a cache.
+3. The shared cache manager (aka. SCM).
+4. The localization service and uploader.
+
+### The Shared Cache Client
+
+YARN application developers and users, should interact with the shared cache using
+the shared cache client. This client is responsible for interacting with the
+shared cache manager, computing the checksum of application resources, and
+claiming application resources in the shared cache. Once an application has claimed
+a resource, it is free to use that resource for the life-cycle of the application.
+Please see the SharedCacheClient.java javadoc for further documentation.
+
+### The Shared Cache HDFS Directory
+
+The shared cache HDFS directory stores all of the shared cache resources. It is protected
+by HDFS permissions and is globally readable, but writing is restricted to a trusted user.
+This HDFS directory is only modified by the shared cache manager and the resource uploader
+on the node manager. Resources are spread across a set of subdirectories using the resources's
+checksum:
+```
+/sharedcache/a/8/9/a896857d078/foo.jar
+/sharedcache/5/0/f/50f11b09f87/bar.jar
+/sharedcache/a/6/7/a678cb1aa8f/job.jar
+```
+
+### Shared Cache Manager (SCM)
+
+The shared cache manager is responsible for serving requests from the client and
+managing the contents of the shared cache. It looks after both the meta data as
+well as the persisted resources in HDFS. It is made up of two major components,
+a back end store and a cleaner service. The SCM runs as a separate daemon
+process that can be placed on any node in the cluster. This allows for
+administrators to start/stop/upgrade the SCM without affecting other YARN
+components (i.e. the resource manager or node managers).
+
+The back end store is responsible for maintaining and persisting metadata about
+the shared cache. This includes the resources in the cache, when a resource was
+last used and a list of applications that are currently using the resource. The
+implementation for the backing store is pluggable and it currently uses an
+in-memory store that recreates its state after a restart.
+
+The cleaner service maintains the persisted resources in HDFS by ensuring that
+resources that are no longer used are removed from the cache. It scans the
+resources in the cache periodically and evicts resources if they are both stale
+and there are no live applications currently using the application.
+
+### The Shared Cache uploader and localization
+
+The shared cache uploader is a service that runs on the node manager and adds
+resources to the shared cache. It is responsible for verifying a resources
+checksum, uploading the resource to HDFS and notifying the shared cache
+manager that a resource has been added to the cache. It is important to note
+that the uploader service is asynchronous from the container launch and does not
+block the startup of a yarn application. In addition adding things to the cache
+is done in a best effort way and does not impact running applications. Once the
+uploader has placed a resource in the shared cache, YARN uses the normal node
+manager localization mechanism to make resources available to the application.
+
+Developing YARN applications with the Shared Cache
+------------
+
+To support the YARN shared cache, an application must use the shared cache
+client during application submission. The shared cache client returns a
+URL corresponding to a resource if it is in the shared cache. To use the cached
+resource, a YARN application simply uses the cached URL to create a
+LocalResource object and sets setShouldBeUploadedToSharedCache to true during
+application submission.
+
+For example, here is how you would create a LocalResource using a cached URL:
+```
+String localPathChecksum = sharedCacheClient.getFileChecksum(localPath);
+URL cachedResource = sharedCacheClient.use(appId, localPathChecksum);
+LocalResource resource = LocalResource.newInstance(cachedResource,
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC
+ size, timestamp, null, true);
+```
+
+Administrating the Shared Cache
+------------
+
+### Setting up the shared cache
+An administrator can initially set up the shared cache by following these steps:
+
+1. Create an HDFS directory for the shared cache (default: /sharedcache).
+2. Set the shared cache directory permissions to 0755.
+3. Ensure that the shared cache directory is owned by the user that runs the
+shared cache manager daemon and the node manager.
+4. In the yarn-site.xml file, set *yarn.sharedcache.enabled* to true and
+*yarn.sharedcache.root-dir* to the directory specified in step 1. For more configuration
+parameters, see the configuration parameters section.
+5. Start the shared cache manager:
+```
+/hadoop/bin/yarn --daemon start sharedcachemanager
+```
+
+### Configuration parameters
+The configuration parameters can be found in yarn-default.xml and should be set
+in the yarn-site.xml file. Here are a list of configuration parameters and their defaults:
+
+Name | Description | Default value
+--- | --- | ---
+yarn.sharedcache.enabled | Whether the shared cache is enabled | false
+yarn.sharedcache.root-dir | The root directory for the shared cache | /sharedcache
+yarn.sharedcache.nested-level | The level of nested directories before getting to the checksum directories. It must be non-negative. | 3
+yarn.sharedcache.store.class | The implementation to be used for the SCM store | org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore
+yarn.sharedcache.app-checker.class | The implementation to be used for the SCM app-checker | org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker
+yarn.sharedcache.store.in-memory.staleness-period-mins | A resource in the in-memory store is considered stale if the time since the last reference exceeds the staleness period. This value is specified in minutes. | 10080
+yarn.sharedcache.store.in-memory.initial-delay-mins | Initial delay before the in-memory store runs its first check to remove dead initial applications. Specified in minutes. | 10
+yarn.sharedcache.store.in-memory.check-period-mins | The frequency at which the in-memory store checks to remove dead initial applications. Specified in minutes. | 720
+yarn.sharedcache.admin.address | The address of the admin interface in the SCM (shared cache manager) | 0.0.0.0:8047
+yarn.sharedcache.admin.thread-count | The number of threads used to handle SCM admin interface (1 by default) | 1
+yarn.sharedcache.webapp.address | The address of the web application in the SCM (shared cache manager) | 0.0.0.0:8788
+yarn.sharedcache.cleaner.period-mins | The frequency at which a cleaner task runs. Specified in minutes. | 1440
+yarn.sharedcache.cleaner.initial-delay-mins | Initial delay before the first cleaner task is scheduled. Specified in minutes. | 10
+yarn.sharedcache.cleaner.resource-sleep-ms | The time to sleep between processing each shared cache resource. Specified in milliseconds. | 0
+yarn.sharedcache.uploader.server.address | The address of the node manager interface in the SCM (shared cache manager) | 0.0.0.0:8046
+yarn.sharedcache.uploader.server.thread-count | The number of threads used to handle shared cache manager requests from the node manager (50 by default) | 50
+yarn.sharedcache.client-server.address | The address of the client interface in the SCM (shared cache manager) | 0.0.0.0:8045
+yarn.sharedcache.client-server.thread-count | The number of threads used to handle shared cache manager requests from clients (50 by default) | 50
+yarn.sharedcache.checksum.algo.impl | The algorithm used to compute checksums of files (SHA-256 by default) | org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl
+yarn.sharedcache.nm.uploader.replication.factor | The replication factor for the node manager uploader for the shared cache (10 by default) | 10
+yarn.sharedcache.nm.uploader.thread-count | The number of threads used to upload files from a node manager instance (20 by default) | 20
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-14521. KMS client needs
retry logic. Contributed by Rushabh S Shah.
Posted by jh...@apache.org.
HADOOP-14521. KMS client needs retry logic. Contributed by Rushabh S Shah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25f31d9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25f31d9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25f31d9f
Branch: refs/heads/YARN-5734
Commit: 25f31d9fc47d21ac2f3afd7042e2ced1b849da39
Parents: 644c2f6
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Oct 5 19:38:00 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Oct 5 19:46:47 2017 -0700
----------------------------------------------------------------------
.../crypto/key/kms/KMSClientProvider.java | 39 ++-
.../key/kms/LoadBalancingKMSClientProvider.java | 81 ++++-
.../fs/CommonConfigurationKeysPublic.java | 29 ++
.../src/main/resources/core-default.xml | 28 ++
.../kms/TestLoadBalancingKMSClientProvider.java | 313 ++++++++++++++++++-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 19 +-
6 files changed, 466 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a95d7e6..c514beb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -250,9 +250,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
* - HOSTNAME = string
* - PORT = integer
*
- * If multiple hosts are provider, the Factory will create a
- * {@link LoadBalancingKMSClientProvider} that round-robins requests
- * across the provided list of hosts.
+ * This will always create a {@link LoadBalancingKMSClientProvider}
+ * if the uri is correct.
*/
@Override
public KeyProvider createProvider(URI providerUri, Configuration conf)
@@ -279,30 +278,26 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
hostsPart = t[0];
}
- return createProvider(providerUri, conf, origUrl, port, hostsPart);
+ return createProvider(conf, origUrl, port, hostsPart);
}
return null;
}
- private KeyProvider createProvider(URI providerUri, Configuration conf,
+ private KeyProvider createProvider(Configuration conf,
URL origUrl, int port, String hostsPart) throws IOException {
String[] hosts = hostsPart.split(";");
- if (hosts.length == 1) {
- return new KMSClientProvider(providerUri, conf);
- } else {
- KMSClientProvider[] providers = new KMSClientProvider[hosts.length];
- for (int i = 0; i < hosts.length; i++) {
- try {
- providers[i] =
- new KMSClientProvider(
- new URI("kms", origUrl.getProtocol(), hosts[i], port,
- origUrl.getPath(), null, null), conf);
- } catch (URISyntaxException e) {
- throw new IOException("Could not instantiate KMSProvider..", e);
- }
+ KMSClientProvider[] providers = new KMSClientProvider[hosts.length];
+ for (int i = 0; i < hosts.length; i++) {
+ try {
+ providers[i] =
+ new KMSClientProvider(
+ new URI("kms", origUrl.getProtocol(), hosts[i], port,
+ origUrl.getPath(), null, null), conf);
+ } catch (URISyntaxException e) {
+ throw new IOException("Could not instantiate KMSProvider.", e);
}
- return new LoadBalancingKMSClientProvider(providers, conf);
}
+ return new LoadBalancingKMSClientProvider(providers, conf);
}
}
@@ -1031,7 +1026,11 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (Exception e) {
- throw new IOException(e);
+ if (e instanceof IOException) {
+ throw (IOException) e;
+ } else {
+ throw new IOException(e);
+ }
}
}
return tokens;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 71d32ff..42cd47d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.crypto.key.kms;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
@@ -31,9 +32,13 @@ import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +74,8 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
private final KMSClientProvider[] providers;
private final AtomicInteger currentIdx;
+ private RetryPolicy retryPolicy = null;
+
public LoadBalancingKMSClientProvider(KMSClientProvider[] providers,
Configuration conf) {
this(shuffle(providers), Time.monotonicNow(), conf);
@@ -80,24 +87,82 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
super(conf);
this.providers = providers;
this.currentIdx = new AtomicInteger((int)(seed % providers.length));
+ int maxNumRetries = conf.getInt(CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, providers.length);
+ int sleepBaseMillis = conf.getInt(CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY,
+ CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT);
+ int sleepMaxMillis = conf.getInt(CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY,
+ CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT);
+ Preconditions.checkState(maxNumRetries >= 0);
+ Preconditions.checkState(sleepBaseMillis >= 0);
+ Preconditions.checkState(sleepMaxMillis >= 0);
+ this.retryPolicy = RetryPolicies.failoverOnNetworkException(
+ RetryPolicies.TRY_ONCE_THEN_FAIL, maxNumRetries, 0, sleepBaseMillis,
+ sleepMaxMillis);
}
@VisibleForTesting
- KMSClientProvider[] getProviders() {
+ public KMSClientProvider[] getProviders() {
return providers;
}
private <T> T doOp(ProviderCallable<T> op, int currPos)
throws IOException {
+ if (providers.length == 0) {
+ throw new IOException("No providers configured !");
+ }
IOException ex = null;
- for (int i = 0; i < providers.length; i++) {
+ int numFailovers = 0;
+ for (int i = 0;; i++, numFailovers++) {
KMSClientProvider provider = providers[(currPos + i) % providers.length];
try {
return op.call(provider);
+ } catch (AccessControlException ace) {
+ // No need to retry on AccessControlException
+ // and AuthorizationException.
+ // This assumes all the servers are configured with identical
+ // permissions and identical key acls.
+ throw ace;
} catch (IOException ioe) {
- LOG.warn("KMS provider at [{}] threw an IOException!! {}",
- provider.getKMSUrl(), StringUtils.stringifyException(ioe));
+ LOG.warn("KMS provider at [{}] threw an IOException: ",
+ provider.getKMSUrl(), ioe);
ex = ioe;
+
+ RetryAction action = null;
+ try {
+ action = retryPolicy.shouldRetry(ioe, 0, numFailovers, false);
+ } catch (Exception e) {
+ if (e instanceof IOException) {
+ throw (IOException)e;
+ }
+ throw new IOException(e);
+ }
+ // make sure each provider is tried at least once, to keep behavior
+ // compatible with earlier versions of LBKMSCP
+ if (action.action == RetryAction.RetryDecision.FAIL
+ && numFailovers >= providers.length - 1) {
+ LOG.warn("Aborting since the Request has failed with all KMS"
+ + " providers(depending on {}={} setting and numProviders={})"
+ + " in the group OR the exception is not recoverable",
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY,
+ getConf().getInt(
+ CommonConfigurationKeysPublic.
+ KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, providers.length),
+ providers.length);
+ throw ex;
+ }
+ if (((numFailovers + 1) % providers.length) == 0) {
+ // Sleep only after we try all the providers for every cycle.
+ try {
+ Thread.sleep(action.delayMillis);
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException("Thread Interrupted");
+ }
+ }
} catch (Exception e) {
if (e instanceof RuntimeException) {
throw (RuntimeException)e;
@@ -106,12 +171,6 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
}
}
}
- if (ex != null) {
- LOG.warn("Aborting since the Request has failed with all KMS"
- + " providers in the group. !!");
- throw ex;
- }
- throw new IOException("No providers configured !!");
}
private int nextIdx() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index b5f355a..4fda2b8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -726,6 +726,35 @@ public class CommonConfigurationKeysPublic {
* <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
* core-default.xml</a>
*/
+ /** Default value is the number of providers specified. */
+ public static final String KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY =
+ "hadoop.security.kms.client.failover.max.retries";
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_KEY =
+ "hadoop.security.kms.client.failover.sleep.base.millis";
+ /** Default value is 100 ms. */
+ public static final int KMS_CLIENT_FAILOVER_SLEEP_BASE_MILLIS_DEFAULT = 100;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_KEY =
+ "hadoop.security.kms.client.failover.sleep.max.millis";
+ /** Default value is 2 secs. */
+ public static final int KMS_CLIENT_FAILOVER_SLEEP_MAX_MILLIS_DEFAULT = 2000;
+
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
public static final String HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY =
"hadoop.security.java.secure.random.algorithm";
/** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index d481048..291608c 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2335,6 +2335,34 @@
</description>
</property>
+<property>
+ <name>hadoop.security.kms.client.failover.sleep.base.millis</name>
+ <value>100</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the base value used in the failover calculation. The
+ first failover will retry immediately. The 2nd failover attempt
+ will delay at least hadoop.security.client.failover.sleep.base.millis
+ milliseconds. And so on.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.kms.client.failover.sleep.max.millis</name>
+ <value>2000</value>
+ <description>
+ Expert only. The time to wait, in milliseconds, between failover
+ attempts increases exponentially as a function of the number of
+ attempts made so far, with a random factor of +/- 50%. This option
+ specifies the maximum value to wait between failovers.
+ Specifically, the time between two failover attempts will not
+ exceed +/- 50% of hadoop.security.client.failover.sleep.max.millis
+ milliseconds.
+ </description>
+</property>
+
<property>
<name>ipc.server.max.connections</name>
<value>0</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
index d14dd59..7a70e18 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java
@@ -23,9 +23,12 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.verify;
import java.io.IOException;
+import java.net.NoRouteToHostException;
import java.net.URI;
+import java.net.UnknownHostException;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
@@ -33,6 +36,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProvider.Options;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.net.ConnectTimeoutException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.junit.Test;
@@ -47,14 +53,17 @@ public class TestLoadBalancingKMSClientProvider {
Configuration conf = new Configuration();
KeyProvider kp = new KMSClientProvider.Factory().createProvider(new URI(
"kms://http@host1/kms/foo"), conf);
- assertTrue(kp instanceof KMSClientProvider);
- assertEquals("http://host1/kms/foo/v1/",
- ((KMSClientProvider) kp).getKMSUrl());
+ assertTrue(kp instanceof LoadBalancingKMSClientProvider);
+ KMSClientProvider[] providers =
+ ((LoadBalancingKMSClientProvider) kp).getProviders();
+ assertEquals(1, providers.length);
+ assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/"),
+ Sets.newHashSet(providers[0].getKMSUrl()));
kp = new KMSClientProvider.Factory().createProvider(new URI(
"kms://http@host1;host2;host3/kms/foo"), conf);
assertTrue(kp instanceof LoadBalancingKMSClientProvider);
- KMSClientProvider[] providers =
+ providers =
((LoadBalancingKMSClientProvider) kp).getProviders();
assertEquals(3, providers.length);
assertEquals(Sets.newHashSet("http://host1/kms/foo/v1/",
@@ -320,4 +329,298 @@ public class TestLoadBalancingKMSClientProvider {
Mockito.verify(p1, Mockito.times(1)).warmUpEncryptedKeys(keyName);
Mockito.verify(p2, Mockito.times(1)).warmUpEncryptedKeys(keyName);
}
-}
+
+ /**
+ * Tests whether retryPolicy fails immediately, after trying each provider
+ * once, on encountering IOException which is not SocketException.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithIOException() throws Exception {
+ Configuration conf = new Configuration();
+ // Setting total failover attempts to .
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 10);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.getMetadata(Mockito.anyString()))
+ .thenThrow(new IOException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.getMetadata(Mockito.anyString()))
+ .thenThrow(new IOException("p2"));
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.getMetadata(Mockito.anyString()))
+ .thenThrow(new IOException("p3"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+ when(p3.getKMSUrl()).thenReturn("p3");
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2, p3}, 0, conf);
+ try {
+ kp.getMetadata("test3");
+ fail("Should fail since all providers threw an IOException");
+ } catch (Exception e) {
+ assertTrue(e instanceof IOException);
+ }
+ verify(kp.getProviders()[0], Mockito.times(1))
+ .getMetadata(Mockito.eq("test3"));
+ verify(kp.getProviders()[1], Mockito.times(1))
+ .getMetadata(Mockito.eq("test3"));
+ verify(kp.getProviders()[2], Mockito.times(1))
+ .getMetadata(Mockito.eq("test3"));
+ }
+
+ /**
+ * Tests that client doesn't retry once it encounters AccessControlException
+ * from first provider.
+ * This assumes all the kms servers are configured with identical access to
+ * keys.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithAccessControlException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new AccessControlException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p2"));
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p3"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+ when(p3.getKMSUrl()).thenReturn("p3");
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2, p3}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ fail("Should fail because provider p1 threw an AccessControlException");
+ } catch (Exception e) {
+ assertTrue(e instanceof AccessControlException);
+ }
+ verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.never()).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p3, Mockito.never()).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests that client doesn't retry once it encounters RunTimeException
+ * from first provider.
+ * This assumes all the kms servers are configured with identical access to
+ * keys.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithRuntimeException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new RuntimeException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException("p2"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ fail("Should fail since provider p1 threw RuntimeException");
+ } catch (Exception e) {
+ assertTrue(e instanceof RuntimeException);
+ }
+ verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.never()).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests the client retries until it finds a good provider.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithTimeoutsException() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 4);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new UnknownHostException("p2"));
+ KMSClientProvider p3 = mock(KMSClientProvider.class);
+ when(p3.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new NoRouteToHostException("p3"));
+ KMSClientProvider p4 = mock(KMSClientProvider.class);
+ when(p4.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenReturn(
+ new KMSClientProvider.KMSKeyVersion("test3", "v1", new byte[0]));
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+ when(p3.getKMSUrl()).thenReturn("p3");
+ when(p4.getKMSUrl()).thenReturn("p4");
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2, p3, p4}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ } catch (Exception e) {
+ fail("Provider p4 should have answered the request.");
+ }
+ verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p3, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p4, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests the operation succeeds second time after ConnectTimeoutException.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesSucceedsSecondTime() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p1"))
+ .thenReturn(new KMSClientProvider.KMSKeyVersion("test3", "v1",
+ new byte[0]));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p2"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ } catch (Exception e) {
+ fail("Provider p1 should have answered the request second time.");
+ }
+ verify(p1, Mockito.times(2)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests whether retryPolicy retries specified number of times.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesSpecifiedNumberOfTimes() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 10);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p2"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ fail("Should fail");
+ } catch (Exception e) {
+ assert (e instanceof ConnectTimeoutException);
+ }
+ verify(p1, Mockito.times(6)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(5)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests whether retryPolicy retries number of times equals to number of
+ * providers if conf kms.client.failover.max.attempts is not set.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesIfMaxAttemptsNotSet() throws Exception {
+ Configuration conf = new Configuration();
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p1"));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new ConnectTimeoutException("p2"));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ fail("Should fail");
+ } catch (Exception e) {
+ assert (e instanceof ConnectTimeoutException);
+ }
+ verify(p1, Mockito.times(2)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+
+ /**
+ * Tests that client reties each provider once, when it encounters
+ * AuthenticationException wrapped in an IOException from first provider.
+ * @throws Exception
+ */
+ @Test
+ public void testClientRetriesWithAuthenticationExceptionWrappedinIOException()
+ throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(
+ CommonConfigurationKeysPublic.KMS_CLIENT_FAILOVER_MAX_RETRIES_KEY, 3);
+ KMSClientProvider p1 = mock(KMSClientProvider.class);
+ when(p1.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException(new AuthenticationException("p1")));
+ KMSClientProvider p2 = mock(KMSClientProvider.class);
+ when(p2.createKey(Mockito.anyString(), Mockito.any(Options.class)))
+ .thenThrow(new IOException(new AuthenticationException("p1")));
+
+ when(p1.getKMSUrl()).thenReturn("p1");
+ when(p2.getKMSUrl()).thenReturn("p2");
+
+ LoadBalancingKMSClientProvider kp = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {p1, p2}, 0, conf);
+ try {
+ kp.createKey("test3", new Options(conf));
+ fail("Should fail since provider p1 threw AuthenticationException");
+ } catch (Exception e) {
+ assertTrue(e.getCause() instanceof AuthenticationException);
+ }
+ verify(p1, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ verify(p2, Mockito.times(1)).createKey(Mockito.eq("test3"),
+ Mockito.any(Options.class));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/25f31d9f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
index 959e724..6f53362 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue;
import com.google.common.base.Supplier;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -69,14 +70,21 @@ public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
protected void setProvider() {
}
+ private KMSClientProvider getKMSClientProvider() {
+ LoadBalancingKMSClientProvider lbkmscp =
+ (LoadBalancingKMSClientProvider) Whitebox
+ .getInternalState(cluster.getNamesystem().getProvider(), "extension");
+ assert lbkmscp.getProviders().length == 1;
+ return lbkmscp.getProviders()[0];
+ }
+
@Test(timeout = 120000)
public void testCreateEZPopulatesEDEKCache() throws Exception {
final Path zonePath = new Path("/TestEncryptionZone");
fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false);
dfsAdmin.createEncryptionZone(zonePath, TEST_KEY, NO_TRASH);
@SuppressWarnings("unchecked")
- KMSClientProvider kcp = (KMSClientProvider) Whitebox
- .getInternalState(cluster.getNamesystem().getProvider(), "extension");
+ KMSClientProvider kcp = getKMSClientProvider();
assertTrue(kcp.getEncKeyQueueSize(TEST_KEY) > 0);
}
@@ -110,8 +118,7 @@ public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
dfsAdmin.createEncryptionZone(zonePath, anotherKey, NO_TRASH);
@SuppressWarnings("unchecked")
- KMSClientProvider spy = (KMSClientProvider) Whitebox
- .getInternalState(cluster.getNamesystem().getProvider(), "extension");
+ KMSClientProvider spy = getKMSClientProvider();
assertTrue("key queue is empty after creating encryption zone",
spy.getEncKeyQueueSize(TEST_KEY) > 0);
@@ -122,9 +129,7 @@ public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
- final KMSClientProvider kspy = (KMSClientProvider) Whitebox
- .getInternalState(cluster.getNamesystem().getProvider(),
- "extension");
+ final KMSClientProvider kspy = getKMSClientProvider();
return kspy.getEncKeyQueueSize(TEST_KEY) > 0;
}
}, 1000, 60000);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-5948. Implement
MutableConfigurationManager for handling storage into configuration store
Posted by jh...@apache.org.
YARN-5948. Implement MutableConfigurationManager for handling storage into configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d95fef9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d95fef9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d95fef9
Branch: refs/heads/YARN-5734
Commit: 8d95fef9f7a1cd98d30f5f1f7a73ec85f93985df
Parents: c1a4709
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Wed Mar 1 16:03:01 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:52 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../src/main/resources/yarn-default.xml | 12 +++
.../scheduler/MutableConfigurationProvider.java | 35 ++++++++
.../scheduler/capacity/CapacityScheduler.java | 14 ++-
.../CapacitySchedulerConfiguration.java | 3 +
.../capacity/conf/CSConfigurationProvider.java | 3 +-
.../conf/MutableCSConfigurationProvider.java | 94 ++++++++++++++++++++
.../conf/YarnConfigurationStoreFactory.java | 46 ++++++++++
.../TestMutableCSConfigurationProvider.java | 83 +++++++++++++++++
9 files changed, 291 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d99941d..92b9697 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -674,6 +674,12 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
+ YARN_PREFIX + "scheduler.configuration.store.class";
+ public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String DEFAULT_CONFIGURATION_STORE =
+ MEMORY_CONFIGURATION_STORE;
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2bc2330..73f4717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3367,4 +3367,16 @@
</description>
</property>
+ <property>
+ <description>
+ The type of configuration store to use for storing scheduler
+ configurations, if using a mutable configuration provider.
+ Keywords such as "memory" map to certain configuration store
+ implementations. If keyword is not found, try to load this
+ value as a class.
+ </description>
+ <name>yarn.scheduler.configuration.store.class</name>
+ <value>memory</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
new file mode 100644
index 0000000..da30a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.Map;
+
+/**
+ * Interface for allowing changing scheduler configurations.
+ */
+public interface MutableConfigurationProvider {
+
+ /**
+ * Update the scheduler configuration with the provided key value pairs.
+ * @param user User issuing the request
+ * @param confUpdate Key-value pairs for configurations to be updated.
+ */
+ void mutateConfiguration(String user, Map<String, String> confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e74cbe1..11f2f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.MutableCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -295,10 +296,15 @@ public class CapacityScheduler extends
String confProviderStr = configuration.get(
CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
- if (confProviderStr.equals(
- CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
- this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
- } else {
+ switch (confProviderStr) {
+ case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ this.csConfProvider =
+ new FileBasedCSConfigurationProvider(rmContext);
+ break;
+ case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
+ break;
+ default:
throw new IOException("Invalid CS configuration provider: " +
confProviderStr);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3821e24..40cb893 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -323,6 +323,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String FILE_CS_CONF_PROVIDER = "file";
@Private
+ public static final String STORE_CS_CONF_PROVIDER = "store";
+
+ @Private
public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
index c9984ac..0d2c8bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -32,8 +32,9 @@ public interface CSConfigurationProvider {
/**
* Initialize the configuration provider with given conf.
* @param conf configuration to initialize with
+ * @throws IOException if initialization fails due to misconfiguration
*/
- void init(Configuration conf);
+ void init(Configuration conf) throws IOException;
/**
* Loads capacity scheduler configuration object.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
new file mode 100644
index 0000000..267ab6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * CS configuration provider which implements
+ * {@link MutableConfigurationProvider} for modifying capacity scheduler
+ * configuration.
+ */
+public class MutableCSConfigurationProvider implements CSConfigurationProvider,
+ MutableConfigurationProvider {
+
+ private Configuration schedConf;
+ private YarnConfigurationStore confStore;
+ private RMContext rmContext;
+ private Configuration conf;
+
+ public MutableCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration config) throws IOException {
+ String store = config.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ this.confStore = new InMemoryConfigurationStore();
+ break;
+ default:
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
+ break;
+ }
+ Configuration initialSchedConf = new Configuration(false);
+ initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+ this.schedConf = initialSchedConf;
+ confStore.initialize(config, initialSchedConf);
+ this.conf = config;
+ }
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration
+ configuration) throws IOException {
+ Configuration loadedConf = new Configuration(configuration);
+ loadedConf.addResource(schedConf);
+ return new CapacitySchedulerConfiguration(loadedConf, false);
+ }
+
+ @Override
+ public void mutateConfiguration(String user,
+ Map<String, String> confUpdate) {
+ Configuration oldConf = new Configuration(schedConf);
+ LogMutation log = new LogMutation(confUpdate, user);
+ long id = confStore.logMutation(log);
+ for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(id, false);
+ return;
+ }
+ confStore.confirmMutation(id, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
new file mode 100644
index 0000000..60249c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of {@link YarnConfigurationStore}.
+ */
+public final class YarnConfigurationStoreFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ YarnConfigurationStoreFactory.class);
+
+ private YarnConfigurationStoreFactory() {
+ // Unused.
+ }
+
+ public static YarnConfigurationStore getStore(Configuration conf) {
+ Class<? extends YarnConfigurationStore> storeClass =
+ conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ InMemoryConfigurationStore.class, YarnConfigurationStore.class);
+ LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d95fef9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
new file mode 100644
index 0000000..3f103b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests {@link MutableCSConfigurationProvider}.
+ */
+public class TestMutableCSConfigurationProvider {
+
+ private MutableCSConfigurationProvider confProvider;
+ private RMContext rmContext;
+ private Map<String, String> goodUpdate;
+ private Map<String, String> badUpdate;
+ private CapacityScheduler cs;
+
+ private static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() {
+ cs = mock(CapacityScheduler.class);
+ rmContext = mock(RMContext.class);
+ when(rmContext.getScheduler()).thenReturn(cs);
+ confProvider = new MutableCSConfigurationProvider(rmContext);
+ goodUpdate = new HashMap<>();
+ goodUpdate.put("goodKey", "goodVal");
+ badUpdate = new HashMap<>();
+ badUpdate.put("badKey", "badVal");
+ }
+
+ @Test
+ public void testInMemoryBackedProvider() throws IOException {
+ Configuration conf = new Configuration();
+ confProvider.init(conf);
+ assertNull(confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ doNothing().when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ assertEquals("goodVal", confProvider.loadConfiguration(conf)
+ .get("goodKey"));
+
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
+ any(RMContext.class));
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-5953 addendum: Move
QueueConfigInfo and SchedConfUpdateInfo to package
org.apache.hadoop.yarn.webapp.dao
Posted by jh...@apache.org.
YARN-5953 addendum: Move QueueConfigInfo and SchedConfUpdateInfo to package org.apache.hadoop.yarn.webapp.dao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/376ab6b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/376ab6b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/376ab6b6
Branch: refs/heads/YARN-5734
Commit: 376ab6b6decc6da752be4d8bcc12c76ff0fbec71
Parents: 605ff23
Author: Xuan <xg...@apache.org>
Authored: Mon Jul 31 11:49:05 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 57 +++++++++++++
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 85 ++++++++++++++++++++
.../webapp/dao/QueueConfigInfo.java | 57 -------------
.../webapp/dao/SchedConfUpdateInfo.java | 85 --------------------
4 files changed, 142 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376ab6b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..d1d91c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376ab6b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
new file mode 100644
index 0000000..bb84096
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.webapp.dao;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue, as well as global scheduler conf changes).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SchedConfUpdateInfo {
+
+ @XmlElement(name = "add-queue")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove-queue")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update-queue")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ private HashMap<String, String> global = new HashMap<>();
+
+ public SchedConfUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
+ this.addQueueInfo = addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
+ this.removeQueueInfo = removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+
+ public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
+ this.updateQueueInfo = updateQueueInfo;
+ }
+
+ @XmlElementWrapper(name = "global-updates")
+ public HashMap<String, String> getGlobalParams() {
+ return global;
+ }
+
+ public void setGlobalParams(HashMap<String, String> globalInfo) {
+ this.global = globalInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376ab6b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
deleted file mode 100644
index d1d91c2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.webapp.dao;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for adding or updating a queue to scheduler configuration
- * for this queue.
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class QueueConfigInfo {
-
- @XmlElement(name = "queueName")
- private String queue;
-
- private HashMap<String, String> params = new HashMap<>();
-
- public QueueConfigInfo() { }
-
- public QueueConfigInfo(String queue, Map<String, String> params) {
- this.queue = queue;
- this.params = new HashMap<>(params);
- }
-
- public String getQueue() {
- return this.queue;
- }
-
- public HashMap<String, String> getParams() {
- return this.params;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376ab6b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
deleted file mode 100644
index bb84096..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.webapp.dao;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElementWrapper;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Information for making scheduler configuration changes (supports adding,
- * removing, or updating a queue, as well as global scheduler conf changes).
- */
-@XmlRootElement(name = "schedConf")
-@XmlAccessorType(XmlAccessType.FIELD)
-public class SchedConfUpdateInfo {
-
- @XmlElement(name = "add-queue")
- private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "remove-queue")
- private ArrayList<String> removeQueueInfo = new ArrayList<>();
-
- @XmlElement(name = "update-queue")
- private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
-
- private HashMap<String, String> global = new HashMap<>();
-
- public SchedConfUpdateInfo() {
- // JAXB needs this
- }
-
- public ArrayList<QueueConfigInfo> getAddQueueInfo() {
- return addQueueInfo;
- }
-
- public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
- this.addQueueInfo = addQueueInfo;
- }
-
- public ArrayList<String> getRemoveQueueInfo() {
- return removeQueueInfo;
- }
-
- public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
- this.removeQueueInfo = removeQueueInfo;
- }
-
- public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
- return updateQueueInfo;
- }
-
- public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
- this.updateQueueInfo = updateQueueInfo;
- }
-
- @XmlElementWrapper(name = "global-updates")
- public HashMap<String, String> getGlobalParams() {
- return global;
- }
-
- public void setGlobalParams(HashMap<String, String> globalInfo) {
- this.global = globalInfo;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: Add 3.0.0-beta1 CHANGES and
RELEASENOTES.
Posted by jh...@apache.org.
Add 3.0.0-beta1 CHANGES and RELEASENOTES.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b34b3ff9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b34b3ff9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b34b3ff9
Branch: refs/heads/YARN-5734
Commit: b34b3ff94baf1d343d60d3f3924e3d28dace13d2
Parents: 79e37dc
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Oct 3 17:14:32 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Oct 3 17:14:40 2017 -0700
----------------------------------------------------------------------
.../release/3.0.0-beta1/CHANGES.3.0.0-beta1.md | 646 +++++++++++++++++++
.../3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md | 377 +++++++++++
2 files changed, 1023 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-14616. Client fails to read
a block with erasure code (XOR,
native) when one of the data block is lost. Contributed by Huafeng Wang.
Posted by jh...@apache.org.
HADOOP-14616. Client fails to read a block with erasure code (XOR, native) when one of the data block is lost. Contributed by Huafeng Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe84cdc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe84cdc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe84cdc9
Branch: refs/heads/YARN-5734
Commit: fe84cdc9daa37af791a23a390e080777090c5e24
Parents: cae1c73
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Oct 5 11:06:09 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Oct 5 11:06:09 2017 +0900
----------------------------------------------------------------------
.../native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe84cdc9/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
index 13f66e2..e8abd8b 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
@@ -58,7 +58,8 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl(
numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits;
chunkSize = (int)dataLen;
- getInputs(env, inputs, inputOffsets, xorDecoder->inputs, numDataUnits);
+ getInputs(env, inputs, inputOffsets, xorDecoder->inputs,
+ numDataUnits + numParityUnits);
getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits);
for (i = 0; i < numDataUnits + numParityUnits; i++) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-14845. Azure wasb:
getFileStatus not making any auth check. Contributed by Sivaguru Sankaridurg
Posted by jh...@apache.org.
HADOOP-14845. Azure wasb: getFileStatus not making any auth check.
Contributed by Sivaguru Sankaridurg
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9288206c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9288206c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9288206c
Branch: refs/heads/YARN-5734
Commit: 9288206cb3c1a39044a8e106436987185ef43ddf
Parents: f702c95
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 5 15:05:23 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 5 15:05:23 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 79 ++++++++-
.../fs/azure/ITestWasbRemoteCallHelper.java | 2 +-
.../TestNativeAzureFileSystemAuthorization.java | 177 +++++++++++++++----
.../ITestAzureFileSystemInstrumentation.java | 6 +-
4 files changed, 220 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9288206c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 5f86f84..2e5241d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -287,7 +287,7 @@ public class NativeAzureFileSystem extends FileSystem {
* @param fs file system on which a file is written.
* @throws IOException Thrown when fail to write file.
*/
- public void writeFile(FileSystem fs) throws IOException {
+ public void writeFile(NativeAzureFileSystem fs) throws IOException {
Path path = getRenamePendingFilePath();
LOG.debug("Preparing to write atomic rename state to {}", path.toString());
OutputStream output = null;
@@ -296,7 +296,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Write file.
try {
- output = fs.create(path);
+ output = fs.createInternal(path, FsPermission.getFileDefault(), false, null);
output.write(contents.getBytes(Charset.forName("UTF-8")));
} catch (IOException e) {
throw new IOException("Unable to write RenamePending file for folder rename from "
@@ -561,7 +561,7 @@ public class NativeAzureFileSystem extends FileSystem {
if (!sourceFolderGone) {
// Make sure the target folder exists.
Path dst = fullPath(dstKey);
- if (!fs.exists(dst)) {
+ if (!fs.existsInternal(dst)) {
fs.mkdirs(dst);
}
@@ -1655,6 +1655,10 @@ public class NativeAzureFileSystem extends FileSystem {
// At this point, we have exclusive access to the source folder
// via the lease, so we will not conflict with an active folder
// rename operation.
+ //
+ // In the secure case, the call to exists will happen in the context
+ // of the user that initiated the operation. In this case, we should
+ // do the auth-check against ranger for the path.
if (!exists(parent)) {
try {
@@ -1750,6 +1754,30 @@ public class NativeAzureFileSystem extends FileSystem {
performAuthCheck(ancestor, WasbAuthorizationOperations.WRITE, "create", absolutePath);
+ return createInternal(f, permission, overwrite, parentFolderLease);
+ }
+
+
+ /**
+ * This is the version of the create call that is meant for internal usage.
+ * This version is not public facing and does not perform authorization checks.
+ * It is used by the public facing create call and by FolderRenamePending to
+ * create the internal -RenamePending.json file.
+ * @param f the path to a file to be created.
+ * @param permission for the newly created file.
+ * @param overwrite specifies if the file should be overwritten.
+ * @param parentFolderLease lease on the parent folder.
+ * @return the output stream used to write data into the newly created file .
+ * @throws IOException if an IO error occurs while attempting to delete the
+ * path.
+ *
+ */
+ protected FSDataOutputStream createInternal(Path f, FsPermission permission,
+ boolean overwrite,
+ SelfRenewingLease parentFolderLease)
+ throws FileAlreadyExistsException, IOException {
+
+ Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
FileMetadata existingMetadata = store.retrieveMetadata(key);
@@ -2589,6 +2617,49 @@ public class NativeAzureFileSystem extends FileSystem {
// Capture the absolute path and the path to key.
Path absolutePath = makeAbsolute(f);
+
+ if (!isRenamePendingFile(absolutePath)) {
+ Path ancestor = getAncestor(absolutePath);
+ if (ancestor.equals(absolutePath) && !ancestor.equals(new Path("/"))) {
+ performAuthCheck(ancestor.getParent(), WasbAuthorizationOperations.READ,
+ "getFileStatus", absolutePath);
+ }
+ else {
+ performAuthCheck(ancestor, WasbAuthorizationOperations.READ,
+ "getFileStatus", absolutePath);
+ }
+ }
+
+ return getFileStatusInternal(f);
+ }
+
+ /**
+ * Checks if a given path exists in the filesystem.
+ * Calls getFileStatusInternal and has the same costs
+ * as the public facing exists call.
+ * This internal version of the exists call does not perform
+ * authorization checks, and is used internally by various filesystem
+ * operations that need to check if the parent/ancestor/path exist.
+ * The idea is to avoid having to configure authorization policies for
+ * these internal calls.
+ * @param f the path to a file or directory.
+ * @return true if path exists; otherwise false.
+ * @throws IOException if an IO error occurs while attempting to check
+ * for existence of the path.
+ *
+ */
+ protected boolean existsInternal(Path f) throws IOException {
+ try {
+ this.getFileStatusInternal(f);
+ return true;
+ } catch (FileNotFoundException fnfe) {
+ return false;
+ }
+ }
+
+ protected FileStatus getFileStatusInternal(Path f) throws FileNotFoundException, IOException {
+
+ Path absolutePath = makeAbsolute(f);
String key = pathToKey(absolutePath);
if (key.length() == 0) { // root always exists
return newDirectory(null, absolutePath);
@@ -2654,7 +2725,7 @@ public class NativeAzureFileSystem extends FileSystem {
// Check if there is a -RenamePending.json file for this folder, and if so,
// redo the rename.
Path absoluteRenamePendingFile = renamePendingFilePath(f);
- if (exists(absoluteRenamePendingFile)) {
+ if (existsInternal(absoluteRenamePendingFile)) {
FolderRenamePending pending =
new FolderRenamePending(absoluteRenamePendingFile, this);
pending.redo();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9288206c/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java
index 062bc36..6f7e699 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbRemoteCallHelper.java
@@ -357,7 +357,7 @@ public class ITestWasbRemoteCallHelper
performop(mockHttpClient);
- int expectedNumberOfInvocations = isAuthorizationCachingEnabled ? 1 : 2;
+ int expectedNumberOfInvocations = isAuthorizationCachingEnabled ? 2 : 3;
Mockito.verify(mockHttpClient, times(expectedNumberOfInvocations)).execute(Mockito.argThat(new HttpGetForServiceLocal()));
Mockito.verify(mockHttpClient, times(expectedNumberOfInvocations)).execute(Mockito.argThat(new HttpGetForService2()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9288206c/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 2129b33..4e7b6fb 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.azure;
+import java.io.FileNotFoundException;
import java.security.PrivilegedExceptionAction;
import java.io.IOException;
@@ -28,6 +29,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assume;
@@ -38,6 +40,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.annotations.VisibleForTesting;
import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
+import static org.apache.hadoop.fs.azure.CachingAuthorizer.KEY_AUTH_SERVICE_CACHING_ENABLE;
import static org.junit.Assert.assertEquals;
/**
@@ -62,6 +65,7 @@ public class TestNativeAzureFileSystemAuthorization
conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true");
conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost/");
conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "user1 , user2");
+ conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "false");
return conf;
}
@@ -131,6 +135,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -156,6 +161,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -183,6 +189,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("create", testPath);
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -208,6 +215,7 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -313,6 +321,7 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true);
/* for rename */
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -344,6 +353,7 @@ public class TestNativeAzureFileSystemAuthorization
/* to create parent dir */
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, false);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -376,6 +386,8 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dir */
authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, false);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -405,6 +417,8 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner("/", WRITE, true); /* to create parent dirs */
authorizer.addAuthRuleForOwner(parentSrcDir.toString(), WRITE, true);
authorizer.addAuthRuleForOwner(parentDstDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentSrcDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDstDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -512,6 +526,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -536,6 +551,7 @@ public class TestNativeAzureFileSystemAuthorization
setExpectedFailureMessage("delete", testPath);
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
fs.create(testPath);
@@ -545,6 +561,7 @@ public class TestNativeAzureFileSystemAuthorization
/* Remove permissions for delete to force failure */
authorizer.deleteAllAuthRules();
authorizer.addAuthRuleForOwner("/", WRITE, false);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -553,6 +570,7 @@ public class TestNativeAzureFileSystemAuthorization
/* Restore permissions to force a successful delete */
authorizer.deleteAllAuthRules();
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
fs.delete(testPath, false);
@@ -569,9 +587,12 @@ public class TestNativeAzureFileSystemAuthorization
public void testFileDeleteAccessWithIntermediateFoldersCheckPositive() throws Throwable {
Path parentDir = new Path("/testDeleteIntermediateFolder");
- Path testPath = new Path(parentDir, "1/2/test.dat");
+ Path childPath = new Path(parentDir, "1/2");
+ Path testPath = new Path(childPath, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true); // for create and delete
+ authorizer.addAuthRuleForOwner(childPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
authorizer.addAuthRuleForOwner("/testDeleteIntermediateFolder*",
WRITE, true); // for recursive delete
fs.updateWasbAuthorizer(authorizer);
@@ -596,12 +617,16 @@ public class TestNativeAzureFileSystemAuthorization
public void testDeleteAuthCheckFailureLeavesFilesUndeleted() throws Throwable {
Path parentDir = new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted");
- Path testPath1 = new Path(parentDir, "child1/test.dat");
- Path testPath2 = new Path(parentDir, "child2/test.dat");
+ Path childPath1 = new Path(parentDir, "child1");
+ Path childPath2 = new Path(parentDir, "child2");
+ Path testPath1 = new Path(childPath1, "test.dat");
+ Path testPath2 = new Path(childPath2, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(childPath2.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -613,20 +638,19 @@ public class TestNativeAzureFileSystemAuthorization
// revoke write on one of the child folders
authorizer.deleteAllAuthRules();
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted",
- WRITE, true);
- authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child2",
- WRITE, true);
- authorizer.addAuthRuleForOwner("/testDeleteAuthCheckFailureLeavesFilesUndeleted/child1",
- WRITE, false);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(childPath2.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(childPath1.toString(), WRITE, false);
+ authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
assertFalse(fs.delete(parentDir, true));
// Assert that only child2 contents are deleted
ContractTestUtils.assertPathExists(fs, "child1 is deleted!", testPath1);
ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!", testPath2);
- ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!",
- new Path("/testDeleteAuthCheckFailureLeavesFilesUndeleted/childPath2"));
+ ContractTestUtils.assertPathDoesNotExist(fs, "child2 exists after deletion!", childPath2);
ContractTestUtils.assertPathExists(fs, "parentDir is deleted!", parentDir);
}
@@ -647,8 +671,8 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.dat");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitPositive",
- WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -684,8 +708,8 @@ public class TestNativeAzureFileSystemAuthorization
parentDir.toString(), testPath.toString()));
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner("/testSingleFileDeleteWithStickyBitNegative",
- WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -700,16 +724,22 @@ public class TestNativeAzureFileSystemAuthorization
dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- authorizer.addAuthRule(parentDir.toString(), WRITE,
- getCurrentUserShortName(), true);
- fs.delete(testPath, true);
- return null;
+ try {
+ authorizer.addAuthRule(parentDir.toString(), WRITE,
+ getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDir.toString(), READ,
+ getCurrentUserShortName(), true);
+ fs.delete(testPath, true);
+ return null;
+ }
+ catch (WasbAuthorizationException wae) {
+ ContractTestUtils.assertPathExists(fs, "testPath should not be deleted!", testPath);
+ throw wae;
+ }
}
});
}
finally {
- ContractTestUtils.assertPathExists(fs, "testPath should not be deleted!", testPath);
-
allowRecursiveDelete(fs, parentDir.toString());
fs.delete(parentDir, true);
}
@@ -724,12 +754,15 @@ public class TestNativeAzureFileSystemAuthorization
public void testRecursiveDeleteSucceedsWithStickybit() throws Throwable {
Path parentDir = new Path("/testRecursiveDeleteSucceedsWithStickybit");
- Path testFilePath = new Path(parentDir, "child/test.dat");
- Path testFolderPath = new Path(parentDir, "child/testDirectory");
+ Path childDir = new Path(parentDir, "child");
+ Path testFilePath = new Path(childDir, "test.dat");
+ Path testFolderPath = new Path(childDir, "testDirectory");
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testRecursiveDeleteSucceedsWithStickybit*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(childDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -759,12 +792,15 @@ public class TestNativeAzureFileSystemAuthorization
public void testRecursiveDeleteFailsWithStickybit() throws Throwable {
Path parentDir = new Path("/testRecursiveDeleteFailsWithStickybit");
- Path testFilePath = new Path(parentDir, "child/test.dat");
- Path testFolderPath = new Path(parentDir, "child/testDirectory");
+ Path childDir = new Path(parentDir, "child");
+ Path testFilePath = new Path(childDir, "test.dat");
+ Path testFolderPath = new Path(childDir, "testDirectory");
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testRecursiveDeleteFailsWithStickybit*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(childDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -784,8 +820,7 @@ public class TestNativeAzureFileSystemAuthorization
@Override
public Void run() throws Exception {
// Add auth rules for dummyuser
- authorizer.addAuthRule("/", WRITE,
- getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/", WRITE, getCurrentUserShortName(), true);
authorizer.addAuthRule("/testRecursiveDeleteFailsWithStickybit*",
WRITE, getCurrentUserShortName(), true);
@@ -821,6 +856,7 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(
"/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -834,10 +870,12 @@ public class TestNativeAzureFileSystemAuthorization
dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- authorizer.addAuthRule("/", WRITE,
- getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/", WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDir.toString(), READ, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(testFolderPath.toString(), READ, getCurrentUserShortName(), true);
authorizer.addAuthRule("/testDeleteSucceedsForOnlyFilesOwnedByUserWithStickybitSet*",
WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
fs.create(testFolderPath); // the folder will have owner as dummyuser
ContractTestUtils.assertPathExists(fs, "folder was not created", testFolderPath);
@@ -876,6 +914,8 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(
"/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -889,8 +929,8 @@ public class TestNativeAzureFileSystemAuthorization
dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- authorizer.addAuthRule("/testDeleteSucceedsForParentDirectoryOwnerUserWithStickybit",
- WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDir.toString(), WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule(parentDir.toString(), READ, getCurrentUserShortName(), true);
fs.create(testFilePath);
ContractTestUtils.assertPathExists(fs, "file was not created", testFilePath);
@@ -920,12 +960,18 @@ public class TestNativeAzureFileSystemAuthorization
public void testDeleteScenarioForRoot() throws Throwable {
Path rootPath = new Path("/");
Path parentDir = new Path("/testDeleteScenarioForRoot");
- Path testPath1 = new Path(parentDir, "child1/test.dat");
- Path testPath2 = new Path(parentDir, "child2/testFolder");
+ Path childPath1 = new Path(parentDir, "child1");
+ Path childPath2 = new Path(parentDir, "child2");
+ Path testPath1 = new Path(childPath1, "test.dat");
+ Path testPath2 = new Path(childPath2, "testFolder");
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner("/testDeleteScenarioForRoot*",
WRITE, true);
+ authorizer.addAuthRuleForOwner(childPath1.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(childPath2.toString(), READ, true);
+ authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -948,13 +994,14 @@ public class TestNativeAzureFileSystemAuthorization
}
/**
- * Positive test for getFileStatus. No permissions are required for getting filestatus.
+ * Positive test for getFileStatus.
* @throws Throwable
*/
@Test
public void testGetFileStatusPositive() throws Throwable {
Path testPath = new Path("/");
+ authorizer.addAuthRuleForOwner("/", READ, true);
ContractTestUtils.assertIsDirectory(fs, testPath);
}
@@ -968,6 +1015,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsAccessCheckPositive/1/2/3");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -990,6 +1038,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testMkdirsWithExistingHierarchyCheckPositive1");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -998,6 +1047,7 @@ public class TestNativeAzureFileSystemAuthorization
/* Don't need permissions to create a directory that already exists */
authorizer.deleteAllAuthRules();
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true); // for assert
fs.mkdirs(testPath);
ContractTestUtils.assertIsDirectory(fs, testPath);
@@ -1022,6 +1072,10 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.addAuthRuleForOwner(childPath1.toString(),
WRITE, true);
+ authorizer.addAuthRuleForOwner(childPath1.getParent().toString(), READ, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
+ authorizer.addAuthRuleForOwner(childPath3.getParent().toString(), READ, true);
+
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1094,10 +1148,10 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path(parentDir, "test.data");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
authorizer.addAuthRuleForOwner(parentDir.toString(), WRITE, true);
// additional rule used for assertPathExists
authorizer.addAuthRuleForOwner(parentDir.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
try {
@@ -1178,6 +1232,7 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerNegative");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
@@ -1212,9 +1267,10 @@ public class TestNativeAzureFileSystemAuthorization
Path testPath = new Path("/testSetOwnerPositive");
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
- String newOwner = "newowner";
+ String newOwner = "user2";
String newGroup = "newgroup";
UserGroupInformation authorisedUser = UserGroupInformation.createUserForTesting(
@@ -1257,6 +1313,7 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
String newOwner = "newowner";
@@ -1303,6 +1360,7 @@ public class TestNativeAzureFileSystemAuthorization
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
+ authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
String owner = null;
@@ -1328,4 +1386,51 @@ public class TestNativeAzureFileSystemAuthorization
fs.delete(testPath, false);
}
}
+
+ /** Test to ensure that the internal RenamePending mechanism
+ * does not make authorization calls.
+ */
+ @Test
+ public void testRenamePendingAuthorizationCalls() throws Throwable {
+ Path testPath = new Path("/testRenamePendingAuthorizationCalls");
+ Path srcPath = new Path(testPath, "srcPath");
+ Path dstPath = new Path(testPath, "dstPath");
+ Path srcFilePath = new Path(srcPath, "file.txt");
+ Path dstFilePath = new Path(dstPath, "file.txt");
+
+ authorizer.addAuthRuleForOwner("/", WRITE, true);
+ /* Remove nextline after fixing createInternal from FolderRenamePending */
+ authorizer.addAuthRuleForOwner(testPath.toString(), WRITE, true);
+ authorizer.addAuthRuleForOwner(srcPath.getParent().toString(), READ, true);
+ authorizer.addAuthRuleForOwner(dstFilePath.getParent().toString(), READ, true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ try {
+ fs.create(srcFilePath);
+
+ String srcKey = fs.pathToKey(srcPath);
+ String dstKey = fs.pathToKey(dstPath);
+
+ // Create a -RenamePendingFile
+ NativeAzureFileSystem.FolderRenamePending renamePending =
+ new NativeAzureFileSystem.FolderRenamePending(srcKey, dstKey, null, fs);
+ renamePending.writeFile(fs);
+
+ // Initiate the pending-rename
+ fs.getFileStatus(srcPath);
+ } catch (FileNotFoundException fnfe) {
+ // This is expected because getFileStatus would complete the pending "rename"
+ // represented by the -RenamePending file.
+ GenericTestUtils.assertExceptionContains(
+ srcPath.toString() + ": No such file or directory.", fnfe
+ );
+
+ // The pending rename should have completed
+ ContractTestUtils.assertPathExists(fs,
+ "dstFilePath does not exist -- pending rename failed", dstFilePath);
+ } finally {
+ allowRecursiveDelete(fs, testPath.toString());
+ fs.delete(testPath, true);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9288206c/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
index 60e24ee..5f08d80 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/ITestAzureFileSystemInstrumentation.java
@@ -343,9 +343,9 @@ public class ITestAzureFileSystemInstrumentation extends AbstractWasbTestBase {
assertFalse(getFileSystem().exists(filePath));
// At the time of writing this code it takes 2 requests/responses to
// check existence, which seems excessive, plus initial request for
- // container check.
+ // container check, plus 2 ancestor checks only in the secure case.
logOpResponseCount("Checking file existence for non-existent file", base);
- base = assertWebResponsesInRange(base, 1, 3);
+ base = assertWebResponsesInRange(base, 1, 5);
// Create an empty file
assertTrue(getFileSystem().createNewFile(filePath));
@@ -354,7 +354,7 @@ public class ITestAzureFileSystemInstrumentation extends AbstractWasbTestBase {
// Check existence again
assertTrue(getFileSystem().exists(filePath));
logOpResponseCount("Checking file existence for existent file", base);
- base = assertWebResponsesInRange(base, 1, 2);
+ base = assertWebResponsesInRange(base, 1, 4);
// Delete the file
assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-12567.
BlockPlacementPolicyRackFaultTolerant fails with racks with very few nodes.
Posted by jh...@apache.org.
HDFS-12567. BlockPlacementPolicyRackFaultTolerant fails with racks with very few nodes.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/644c2f69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/644c2f69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/644c2f69
Branch: refs/heads/YARN-5734
Commit: 644c2f6924f341f51d809c91dccfff88fc82f6f0
Parents: c071aad
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 5 16:58:43 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Oct 5 16:58:43 2017 -0700
----------------------------------------------------------------------
.../blockmanagement/BlockPlacementPolicy.java | 2 +-
.../BlockPlacementPolicyRackFaultTolerant.java | 49 +++++++--
.../hdfs/TestErasureCodingMultipleRacks.java | 107 +++++++++++++++++++
3 files changed, 146 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/644c2f69/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 732a2dc..23e3e40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
public abstract class BlockPlacementPolicy {
- static final Logger LOG = LoggerFactory.getLogger(
+ public static final Logger LOG = LoggerFactory.getLogger(
BlockPlacementPolicy.class);
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/644c2f69/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
index c0d981c..1eac3ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolerant.java
@@ -46,9 +46,12 @@ public class BlockPlacementPolicyRackFaultTolerant extends BlockPlacementPolicyD
if (numOfRacks == 1 || totalNumOfReplicas <= 1) {
return new int[] {numOfReplicas, totalNumOfReplicas};
}
- if(totalNumOfReplicas<numOfRacks){
+ // If more racks than replicas, put one replica per rack.
+ if (totalNumOfReplicas < numOfRacks) {
return new int[] {numOfReplicas, 1};
}
+ // If more replicas than racks, evenly spread the replicas.
+ // This calculation rounds up.
int maxNodesPerRack = (totalNumOfReplicas - 1) / numOfRacks + 1;
return new int[] {numOfReplicas, maxNodesPerRack};
}
@@ -109,18 +112,42 @@ public class BlockPlacementPolicyRackFaultTolerant extends BlockPlacementPolicyD
numOfReplicas = Math.min(totalReplicaExpected - results.size(),
(maxNodesPerRack -1) * numOfRacks - (results.size() - excess));
- // Fill each rack exactly (maxNodesPerRack-1) replicas.
- writer = chooseOnce(numOfReplicas, writer, new HashSet<>(excludedNodes),
- blocksize, maxNodesPerRack -1, results, avoidStaleNodes, storageTypes);
+ try {
+ // Try to spread the replicas as evenly as possible across racks.
+ // This is done by first placing with (maxNodesPerRack-1), then spreading
+ // the remainder by calling again with maxNodesPerRack.
+ writer = chooseOnce(numOfReplicas, writer, new HashSet<>(excludedNodes),
+ blocksize, maxNodesPerRack - 1, results, avoidStaleNodes,
+ storageTypes);
- for (DatanodeStorageInfo resultStorage : results) {
- addToExcludedNodes(resultStorage.getDatanodeDescriptor(), excludedNodes);
- }
+ // Exclude the chosen nodes
+ for (DatanodeStorageInfo resultStorage : results) {
+ addToExcludedNodes(resultStorage.getDatanodeDescriptor(),
+ excludedNodes);
+ }
+ LOG.trace("Chosen nodes: {}", results);
+ LOG.trace("Excluded nodes: {}", excludedNodes);
- // For some racks, place one more replica to each one of them.
- numOfReplicas = totalReplicaExpected - results.size();
- chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ numOfReplicas = totalReplicaExpected - results.size();
+ chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ } catch (NotEnoughReplicasException e) {
+ LOG.debug("Only able to place {} of {} (maxNodesPerRack={}) nodes " +
+ "evenly across racks, falling back to uneven placement.",
+ results.size(), numOfReplicas, maxNodesPerRack);
+ LOG.debug("Caught exception was:", e);
+ // Exclude the chosen nodes
+ for (DatanodeStorageInfo resultStorage : results) {
+ addToExcludedNodes(resultStorage.getDatanodeDescriptor(),
+ excludedNodes);
+ }
+
+ LOG.trace("Chosen nodes: {}", results);
+ LOG.trace("Excluded nodes: {}", excludedNodes);
+ numOfReplicas = totalReplicaExpected - results.size();
+ chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
+ totalReplicaExpected, results, avoidStaleNodes, storageTypes);
+ }
return writer;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/644c2f69/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
new file mode 100644
index 0000000..eb6213a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingMultipleRacks.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+/**
+ * Test erasure coding block placement with skewed # nodes per rack.
+ */
+public class TestErasureCodingMultipleRacks {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestErasureCodingMultipleRacks.class);
+
+ static {
+ GenericTestUtils.setLogLevel(BlockPlacementPolicy.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(BlockPlacementPolicyDefault.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(BlockPlacementPolicyRackFaultTolerant.LOG,
+ Level.DEBUG);
+ GenericTestUtils.setLogLevel(NetworkTopology.LOG, Level.DEBUG);
+ }
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(300000);
+
+ public ErasureCodingPolicy getPolicy() {
+ return StripedFileTestUtil.getDefaultECPolicy();
+ }
+
+ private MiniDFSCluster cluster;
+ private ErasureCodingPolicy ecPolicy;
+ private Configuration conf;
+ private DistributedFileSystem dfs;
+
+ @Before
+ public void setup() throws Exception {
+ ecPolicy = getPolicy();
+ final int dataUnits = ecPolicy.getNumDataUnits();
+ final int parityUnits = ecPolicy.getNumParityUnits();
+ final int numDatanodes = dataUnits + parityUnits;
+ final int numRacks = 2;
+ final String[] racks = new String[numDatanodes];
+ for (int i = 0; i < numRacks; i++) {
+ racks[i] = "/rack" + i;
+ }
+ for (int i = numRacks; i < numDatanodes; i++) {
+ racks[i] = "/rack" + (numRacks - 1);
+ }
+ conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numDatanodes)
+ .racks(racks)
+ .build();
+ dfs = cluster.getFileSystem();
+ cluster.waitActive();
+ dfs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName());
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testSkewedRack() throws Exception {
+ final int filesize = ecPolicy.getNumDataUnits() * ecPolicy
+ .getCellSize();
+ byte[] contents = new byte[filesize];
+
+ for (int i = 0; i < 10; i++) {
+ final Path path = new Path("/testfile" + i);
+ LOG.info("Writing file " + path);
+ DFSTestUtil.writeFile(dfs, path, contents);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-7024: Fix issues on recovery
in LevelDB store. Contributed by Jonathan Hung
Posted by jh...@apache.org.
YARN-7024: Fix issues on recovery in LevelDB store. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33f10741
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33f10741
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33f10741
Branch: refs/heads/YARN-5734
Commit: 33f1074123d82b9fa118873b5632d4be245d0bcf
Parents: dc53a0c
Author: Xuan <xg...@apache.org>
Authored: Wed Aug 23 11:11:41 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/conf/InMemoryConfigurationStore.java | 2 +-
.../scheduler/capacity/conf/LeveldbConfigurationStore.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33f10741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index b97be1b..c63734d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -79,7 +79,7 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
@Override
public synchronized List<LogMutation> getPendingMutations() {
- return pendingMutations;
+ return new LinkedList<>(pendingMutations);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33f10741/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1534685..1280fab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -65,6 +65,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
private static final String LOG_COMMITTED_TXN = "committedTxn";
private DB db;
+ // Txnid for the last transaction logged to the store.
private long txnId = 0;
private long minTxn = 0;
private long maxLogs;
@@ -92,6 +93,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
break;
}
pendingMutations.add(deserLogMutation(entry.getValue()));
+ txnId++;
}
// Get the earliest txnId stored in logs
itr.seekToFirst();
@@ -278,7 +280,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
@Override
public List<LogMutation> getPendingMutations() {
- return pendingMutations;
+ return new LinkedList<>(pendingMutations);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-5952. Create REST API for
changing YARN scheduler configurations. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
YARN-5952. Create REST API for changing YARN scheduler configurations. (Jonathan Hung via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4006c67b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4006c67b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4006c67b
Branch: refs/heads/YARN-5734
Commit: 4006c67b26a7d0426b7b10551a47506a0c3f366e
Parents: 8d95fef
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Apr 3 10:12:01 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfScheduler.java | 40 ++
.../scheduler/MutableConfigurationProvider.java | 5 +-
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../conf/InMemoryConfigurationStore.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 24 +-
.../resourcemanager/webapp/RMWebServices.java | 172 ++++++-
.../webapp/dao/QueueConfigInfo.java | 57 +++
.../webapp/dao/QueueConfigsUpdateInfo.java | 60 +++
.../TestMutableCSConfigurationProvider.java | 6 +-
.../TestRMWebServicesConfigurationMutation.java | 477 +++++++++++++++++++
10 files changed, 851 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
new file mode 100644
index 0000000..35e36e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface for a scheduler that supports changing configuration at runtime.
+ *
+ */
+public interface MutableConfScheduler extends ResourceScheduler {
+
+ /**
+ * Update the scheduler's configuration.
+ * @param user Caller of this update
+ * @param confUpdate key-value map of the configuration update
+ * @throws IOException if update is invalid
+ */
+ void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index da30a2b..889c3bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.io.IOException;
import java.util.Map;
/**
@@ -29,7 +30,9 @@ public interface MutableConfigurationProvider {
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
+ * @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate);
+ void mutateConfiguration(String user, Map<String, String> confUpdate)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 11f2f6e..7c5839b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -86,6 +86,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -150,7 +152,7 @@ import com.google.common.util.concurrent.SettableFuture;
public class CapacityScheduler extends
AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> implements
PreemptableResourceScheduler, CapacitySchedulerContext, Configurable,
- ResourceAllocationCommitter {
+ ResourceAllocationCommitter, MutableConfScheduler {
private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
@@ -2610,4 +2612,16 @@ public class CapacityScheduler extends
// In seconds
return ((LeafQueue) queue).getMaximumApplicationLifetime();
}
+
+ @Override
+ public void updateConfiguration(UserGroupInformation user,
+ Map<String, String> confUpdate) throws IOException {
+ if (csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
+ user.getShortUserName(), confUpdate);
+ } else {
+ throw new UnsupportedOperationException("Configured CS configuration " +
+ "provider does not support updating configuration.");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index a208fb9..b97be1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -58,7 +58,11 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
if (isValid) {
Map<String, String> mutations = mutation.getUpdates();
for (Map.Entry<String, String> kv : mutations.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 267ab6a..ea1b3c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -60,34 +60,44 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
- this.schedConf = initialSchedConf;
- confStore.initialize(config, initialSchedConf);
+ this.schedConf = new Configuration(false);
+ // We need to explicitly set the key-values in schedConf, otherwise
+ // these configuration keys cannot be deleted when
+ // configuration is reloaded.
+ for (Map.Entry<String, String> kv : initialSchedConf) {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ confStore.initialize(config, schedConf);
this.conf = config;
}
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
- Configuration loadedConf = new Configuration(configuration);
- loadedConf.addResource(schedConf);
+ Configuration loadedConf = new Configuration(schedConf);
+ loadedConf.addResource(configuration);
return new CapacitySchedulerConfiguration(loadedConf, false);
}
@Override
public void mutateConfiguration(String user,
- Map<String, String> confUpdate) {
+ Map<String, String> confUpdate) throws IOException {
Configuration oldConf = new Configuration(schedConf);
LogMutation log = new LogMutation(confUpdate, user);
long id = confStore.logMutation(log);
for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
- schedConf.set(kv.getKey(), kv.getValue());
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
}
try {
rmContext.getScheduler().reinitialize(conf, rmContext);
} catch (IOException e) {
schedConf = oldConf;
confStore.confirmMutation(id, false);
- return;
+ throw e;
}
confStore.confirmMutation(id, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6dc3d9a..215e511 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -55,7 +55,8 @@ import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
-
+import com.google.common.base.Joiner;
+import org.apache.commons.codec.binary.Base64;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -134,11 +135,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2454,4 +2458,170 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
GetContainersRequest request) throws YarnException, IOException {
return rm.getClientRMService().getContainers(request).getContainerList();
}
+
+ @PUT
+ @Path("/queues")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response updateSchedulerConfiguration(QueueConfigsUpdateInfo
+ mutationInfo, @Context HttpServletRequest hsr)
+ throws AuthorizationException, InterruptedException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ ApplicationACLsManager aclsManager = rm.getApplicationACLsManager();
+ if (aclsManager.areACLsEnabled()) {
+ if (callerUGI == null || !aclsManager.isAdmin(callerUGI)) {
+ String msg = "Only admins can carry out this operation.";
+ throw new ForbiddenException(msg);
+ }
+ }
+
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ if (scheduler instanceof MutableConfScheduler) {
+ try {
+ callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException, YarnException {
+ Map<String, String> confUpdate =
+ constructKeyValueConfUpdate(mutationInfo);
+ ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
+ confUpdate);
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
+ .build();
+ }
+ return Response.status(Status.OK).entity("Configuration change " +
+ "successfully applied.").build();
+ } else {
+ return Response.status(Status.BAD_REQUEST)
+ .entity("Configuration change only supported by CapacityScheduler.")
+ .build();
+ }
+ }
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacitySchedulerConfiguration currentConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(currentConf, false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
+ parentQueuePath + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
+ queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
new file mode 100644
index 0000000..b20eda6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for adding or updating a queue to scheduler configuration
+ * for this queue.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigInfo {
+
+ @XmlElement(name = "queueName")
+ private String queue;
+
+ private HashMap<String, String> params = new HashMap<>();
+
+ public QueueConfigInfo() { }
+
+ public QueueConfigInfo(String queue, Map<String, String> params) {
+ this.queue = queue;
+ this.params = new HashMap<>(params);
+ }
+
+ public String getQueue() {
+ return this.queue;
+ }
+
+ public HashMap<String, String> getParams() {
+ return this.params;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
new file mode 100644
index 0000000..644ec90
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigsUpdateInfo.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * Information for making scheduler configuration changes (supports adding,
+ * removing, or updating a queue).
+ */
+@XmlRootElement(name = "schedConf")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueConfigsUpdateInfo {
+
+ @XmlElement(name = "add")
+ private ArrayList<QueueConfigInfo> addQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "remove")
+ private ArrayList<String> removeQueueInfo = new ArrayList<>();
+
+ @XmlElement(name = "update")
+ private ArrayList<QueueConfigInfo> updateQueueInfo = new ArrayList<>();
+
+ public QueueConfigsUpdateInfo() {
+ // JAXB needs this
+ }
+
+ public ArrayList<QueueConfigInfo> getAddQueueInfo() {
+ return addQueueInfo;
+ }
+
+ public ArrayList<String> getRemoveQueueInfo() {
+ return removeQueueInfo;
+ }
+
+ public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
+ return updateQueueInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3f103b1..254da31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -77,7 +77,11 @@ public class TestMutableCSConfigurationProvider {
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ try {
+ confProvider.mutateConfiguration(TEST_USER, badUpdate);
+ } catch (IOException e) {
+ // Expected exception.
+ }
assertNull(confProvider.loadConfiguration(conf).get("badKey"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4006c67b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
new file mode 100644
index 0000000..d149055
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -0,0 +1,477 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Test scheduler configuration mutation via REST API.
+ */
+public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+
+ private static final File CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
+ private static final File OLD_CONF_FILE = new File(new File("target",
+ "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE + ".tmp");
+
+ private static MockRM rm;
+ private static String userName;
+ private static CapacitySchedulerConfiguration csConf;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Unable to get current user name "
+ + ioe.getMessage(), ioe);
+ }
+ csConf = new CapacitySchedulerConfiguration(new Configuration(false),
+ false);
+ setupQueueConfiguration(csConf);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+ try {
+ if (CONF_FILE.exists()) {
+ if (!CONF_FILE.renameTo(OLD_CONF_FILE)) {
+ throw new RuntimeException("Failed to rename conf file");
+ }
+ }
+ FileOutputStream out = new FileOutputStream(CONF_FILE);
+ csConf.writeXml(out);
+ out.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write XML file", e);
+ }
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ filter("/*").through(TestRMWebServicesAppsModification
+ .TestRMCustomAuthFilter.class);
+ }
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ }
+
+ private static void setupQueueConfiguration(
+ CapacitySchedulerConfiguration config) {
+ config.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{"a", "b", "c"});
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ config.setCapacity(a, 25f);
+ config.setMaximumCapacity(a, 50f);
+
+ final String a1 = a + ".a1";
+ final String a2 = a + ".a2";
+ config.setQueues(a, new String[]{"a1", "a2"});
+ config.setCapacity(a1, 100f);
+ config.setCapacity(a2, 0f);
+
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ config.setCapacity(b, 75f);
+
+ final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+ config.setCapacity(c, 0f);
+
+ final String c1 = c + ".c1";
+ config.setQueues(c, new String[] {"c1"});
+ config.setCapacity(c1, 0f);
+ }
+
+ public TestRMWebServicesConfigurationMutation() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testAddNestedQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add parent queue root.d with two children d1 and d2.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> d1Capacity = new HashMap<>();
+ d1Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ d1Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "25");
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY,
+ "1E-4");
+ Map<String, String> d2Capacity = new HashMap<>();
+ d2Capacity.put(CapacitySchedulerConfiguration.CAPACITY, "75");
+ d2Capacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "75");
+ QueueConfigInfo d1 = new QueueConfigInfo("root.d.d1", d1Capacity);
+ QueueConfigInfo d2 = new QueueConfigInfo("root.d.d2", d2Capacity);
+ QueueConfigInfo d = new QueueConfigInfo("root.d", nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d1);
+ updateInfo.getAddQueueInfo().add(d2);
+ updateInfo.getAddQueueInfo().add(d);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(2, newCSConf.getQueues("root.d").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d1"),
+ 0.01f);
+ assertEquals(75.0f, newCSConf.getNonLabeledQueueCapacity("root.d.d2"),
+ 0.01f);
+ }
+
+ @Test
+ public void testAddWithUpdate() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add root.d with capacity 25, reducing root.b capacity from 75 to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> dCapacity = new HashMap<>();
+ dCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "25");
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo d = new QueueConfigInfo("root.d", dCapacity);
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(4, newCSConf.getQueues("root").length);
+ assertEquals(25.0f, newCSConf.getNonLabeledQueueCapacity("root.d"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Test
+ public void testRemoveQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a.a2");
+ // Remove root.a.a2
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a.a2");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root.a").length);
+ assertEquals("a1", newCSConf.getQueues("root.a")[0]);
+ }
+
+ @Test
+ public void testRemoveParentQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.c", "root.c.c1");
+ // Remove root.c (parent queue)
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertNull(newCSConf.getQueues("root.c"));
+ }
+
+ @Test
+ public void testRemoveParentQueueWithCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.a", "root.a.a1", "root.a.a2");
+ // Remove root.a (parent queue) with capacity 25
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.a");
+
+ // Set root.b capacity to 100
+ Map<String, String> bCapacity = new HashMap<>();
+ bCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ QueueConfigInfo b = new QueueConfigInfo("root.b", bCapacity);
+ updateInfo.getUpdateQueueInfo().add(b);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(2, newCSConf.getQueues("root").length);
+ assertEquals(100.0f, newCSConf.getNonLabeledQueueCapacity("root.b"),
+ 0.01f);
+ }
+
+ @Test
+ public void testRemoveMultipleQueues() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ stopQueue("root.b", "root.c", "root.c.c1");
+ // Remove root.b and root.c
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b");
+ updateInfo.getRemoveQueueInfo().add("root.c");
+ Map<String, String> aCapacity = new HashMap<>();
+ aCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ aCapacity.put(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY, "100");
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", aCapacity);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(1, newCSConf.getQueues("root").length);
+ }
+
+ private void stopQueue(String... queuePaths) throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Set state of queues to STOPPED.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ for (String queue : queuePaths) {
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo(queue, stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+ }
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ for (String queue : queuePaths) {
+ assertEquals(QueueState.STOPPED, newCSConf.getState(queue));
+ }
+ }
+
+ @Test
+ public void testUpdateQueue() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update config value.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ "0.2");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
+ cs.getConfiguration()
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
+ assertEquals(0.2f, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"), 0.001f);
+
+ // Remove config. Config value should be reverted to default.
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_AM_RESOURCE_SUFFIX,
+ null);
+ aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().clear();
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf = cs.getConfiguration();
+ assertEquals(CapacitySchedulerConfiguration
+ .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT, newCSConf
+ .getMaximumApplicationMasterResourcePerQueuePercent("root.a"),
+ 0.001f);
+ }
+
+ @Test
+ public void testUpdateQueueCapacity() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Update root.a and root.b capacity to 50.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.CAPACITY, "50");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ QueueConfigInfo bUpdateInfo = new QueueConfigInfo("root.b", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+ updateInfo.getUpdateQueueInfo().add(bUpdateInfo);
+
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("queues").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(updateInfo, QueueConfigsUpdateInfo.class),
+ MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.a"), 0.01f);
+ assertEquals(50.0f, newCSConf.getNonLabeledQueueCapacity("root.b"), 0.01f);
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ if (rm != null) {
+ rm.stop();
+ }
+ CONF_FILE.delete();
+ if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+ throw new RuntimeException("Failed to re-copy old configuration file");
+ }
+ super.tearDown();
+ }
+
+ @SuppressWarnings("rawtypes")
+ private String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-6182. Fix alignment issues
and missing information in new YARN UI's Queue page. Contributed by Akhil PB.
Posted by jh...@apache.org.
YARN-6182. Fix alignment issues and missing information in new YARN UI's Queue page. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f702c957
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f702c957
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f702c957
Branch: refs/heads/YARN-5734
Commit: f702c9575495e3af99d4b5cae9f73969e1a6dfc2
Parents: 592bf2d
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 5 11:34:44 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 5 11:34:44 2017 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/components/nodes-heatmap.js | 1 -
.../src/main/webapp/app/components/tree-selector.js | 4 ++--
.../src/main/webapp/app/controllers/application.js | 9 ++++++++-
.../src/main/webapp/app/models/yarn-queue/capacity-queue.js | 8 +++++---
.../src/main/webapp/app/models/yarn-queue/fair-queue.js | 8 +++++---
.../src/main/webapp/app/models/yarn-queue/fifo-queue.js | 8 +++++---
.../main/webapp/app/serializers/yarn-queue/fair-queue.js | 5 +++--
.../src/main/webapp/app/templates/application.hbs | 2 +-
.../components/yarn-queue/capacity-queue-conf-table.hbs | 4 +++-
.../components/yarn-queue/fair-queue-conf-table.hbs | 2 +-
.../app/templates/components/yarn-queue/fair-queue.hbs | 2 +-
.../components/yarn-queue/fifo-queue-conf-table.hbs | 4 +++-
.../app/templates/components/yarn-queue/fifo-queue.hbs | 2 +-
.../hadoop-yarn-ui/src/main/webapp/app/utils/converter.js | 6 ++++++
14 files changed, 44 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
index 84ff59e..1ea655b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
@@ -258,7 +258,6 @@ export default BaseChartComponent.extend({
},
didInsertElement: function () {
- var parentId = this.get("parentId");
var self = this;
var optionsData = [this.memoryLabel, this.cpuLabel, this.containersLabel];
d3.select("#heatmap-select")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
index 6788582..4a0e6aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
@@ -153,7 +153,7 @@ export default Ember.Component.extend({
.attr("r", 1e-6)
.style("fill", function(d) {
var maxCap = d.queueData.get(this.max);
- maxCap = maxCap == undefined ? 100 : maxCap;
+ maxCap = maxCap === undefined ? 100 : maxCap;
var usedCap = d.queueData.get(this.used) / maxCap * 100.0;
if (usedCap <= 60.0) {
return "LimeGreen";
@@ -172,7 +172,7 @@ export default Ember.Component.extend({
.attr("text-anchor", function() { return "middle"; })
.text(function(d) {
var maxCap = d.queueData.get(this.max);
- maxCap = maxCap == undefined ? 100 : maxCap;
+ maxCap = maxCap === undefined ? 100 : maxCap;
var usedCap = d.queueData.get(this.used) / maxCap * 100.0;
if (usedCap >= 100.0) {
return usedCap.toFixed(0) + "%";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 08ca5a6..986b1fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -50,6 +50,13 @@ export default Ember.Controller.extend({
html = html + '><a href="yarn-nodes">Nodes<span class="sr-only">' +
'(current)</span></a></li>';
return Ember.String.htmlSafe(html);
+ }.property('currentPath'),
+
+ isQueuesTabActive: function() {
+ var path = this.get('currentPath');
+ if (path === 'yarn-queues') {
+ return true;
+ }
+ return false;
}.property('currentPath')
});
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
index 1cb07bb..b84a473 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
@@ -17,6 +17,7 @@
*/
import DS from 'ember-data';
+import Converter from 'yarn-ui/utils/converter';
export default DS.Model.extend({
name: DS.attr('string'),
@@ -46,18 +47,19 @@ export default DS.Model.extend({
}.property("children"),
capacitiesBarChartData: function() {
+ var floatToFixed = Converter.floatToFixed;
return [
{
label: "Absolute Capacity",
- value: this.get("name") === "root" ? 100 : this.get("absCapacity")
+ value: this.get("name") === "root" ? 100 : floatToFixed(this.get("absCapacity"))
},
{
label: "Absolute Used",
- value: this.get("name") === "root" ? this.get("usedCapacity") : this.get("absUsedCapacity")
+ value: this.get("name") === "root" ? floatToFixed(this.get("usedCapacity")) : floatToFixed(this.get("absUsedCapacity"))
},
{
label: "Absolute Max Capacity",
- value: this.get("name") === "root" ? 100 : this.get("absMaxCapacity")
+ value: this.get("name") === "root" ? 100 : floatToFixed(this.get("absMaxCapacity"))
}
];
}.property("absCapacity", "usedCapacity", "absMaxCapacity"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
index be71362..3f358cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
@@ -17,6 +17,7 @@
*/
import DS from 'ember-data';
+import Converter from 'yarn-ui/utils/converter';
export default DS.Model.extend({
name: DS.attr('string'),
@@ -48,18 +49,19 @@ export default DS.Model.extend({
}.property("children"),
capacitiesBarChartData: function() {
+ var floatToFixed = Converter.floatToFixed;
return [
{
label: "Steady Fair Memory",
- value: this.get("steadyFairResources.memory")
+ value: floatToFixed(this.get("steadyFairResources.memory"))
},
{
label: "Used Memory",
- value: this.get("usedResources.memory")
+ value: floatToFixed(this.get("usedResources.memory"))
},
{
label: "Maximum Memory",
- value: this.get("maxResources.memory")
+ value: floatToFixed(this.get("maxResources.memory"))
}
];
}.property("maxResources.memory", "usedResources.memory", "maxResources.memory"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
index 2386dc4..40a8e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
@@ -17,6 +17,7 @@
*/
import DS from 'ember-data';
+import Converter from 'yarn-ui/utils/converter';
export default DS.Model.extend({
name: DS.attr('string'),
@@ -33,18 +34,19 @@ export default DS.Model.extend({
type: DS.attr('string'),
capacitiesBarChartData: function() {
+ var floatToFixed = Converter.floatToFixed;
return [
{
label: "Available Capacity",
- value: this.get("availNodeCapacity")
+ value: floatToFixed(this.get("availNodeCapacity"))
},
{
label: "Used Capacity",
- value: this.get("usedNodeCapacity")
+ value: floatToFixed(this.get("usedNodeCapacity"))
},
{
label: "Total Capacity",
- value: this.get("totalNodeCapacity")
+ value: floatToFixed(this.get("totalNodeCapacity"))
}
];
}.property("availNodeCapacity", "usedNodeCapacity", "totalNodeCapacity")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
index 2215d2d..c730e44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
@@ -59,8 +59,9 @@ export default DS.JSONAPISerializer.extend({
handleQueue(store, primaryModelClass, payload, id, requestType) {
var data = [];
- var includedData = [];
- if(!payload) return data;
+ if (!payload) {
+ return data;
+ }
var result = this.normalizeSingleResponse(store, primaryModelClass,
payload, id, requestType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/application.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/application.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/application.hbs
index e988e0c..1ac53bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/application.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/application.hbs
@@ -41,7 +41,7 @@
<span class="sr-only">(current)</span>
{{/link-to}}
{{/link-to}}
- {{#link-to 'yarn-queues' 'root' tagName="li"}}
+ {{#link-to 'yarn-queues' 'root' tagName="li" classNameBindings="isQueuesTabActive:active"}}
{{#link-to 'yarn-queues' 'root' class="navigation-link"}}Queues
<span class="sr-only">(current)</span>
{{/link-to}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
index 3f6017f..a04a26d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
@@ -20,7 +20,7 @@
<thead>
<tr>
<td><b>Configurations</b></td>
- <td>Value</td>
+ <td><b>Value</b></td>
</tr>
</thead>
<tbody>
@@ -32,10 +32,12 @@
<td>Configured Max Capacity</td>
<td>{{queue.maxCapacity}}</td>
</tr>
+ {{#if queue.state}}
<tr>
<td>State</td>
<td>{{queue.state}}</td>
</tr>
+ {{/if}}
{{#if queue.isLeafQueue}}
<tr>
<td>User Limit Percent</td>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
index 00fabcc..b2448cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
@@ -20,7 +20,7 @@
<thead>
<tr>
<td><b>Configurations</b></td>
- <td>Value</td>
+ <td><b>Value</b></td>
</tr>
</thead>
<tbody>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
index 0341108..6d0e994 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
@@ -39,7 +39,7 @@
{{bar-chart data=model.selectedQueue.capacitiesBarChartData
title=""
parentId="capacity-bar-chart"
- textWidth=150
+ textWidth=175
ratio=0.55
maxHeight=350}}
</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
index 4ced3e7..f6f467f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
@@ -20,7 +20,7 @@
<thead>
<tr>
<td><b>Configurations</b></td>
- <td>Value</td>
+ <td><b>Value</b></td>
</tr>
</thead>
<tbody>
@@ -32,10 +32,12 @@
<td>Used Capacity</td>
<td>{{queue.usedCapacity}}</td>
</tr>
+ {{#if queue.state}}
<tr>
<td>State</td>
<td>{{queue.state}}</td>
</tr>
+ {{/if}}
<tr>
<td>Minimum Queue Memory Capacity</td>
<td>{{queue.minQueueMemoryCapacity}}</td>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
index 46d79f0..90cbd27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
@@ -39,7 +39,7 @@
{{bar-chart data=model.selectedQueue.capacitiesBarChartData
title=""
parentId="capacity-bar-chart"
- textWidth=150
+ textWidth=175
ratio=0.55
maxHeight=350}}
</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f702c957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
index 7c9a1f8..b9da222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
@@ -165,5 +165,11 @@ export default {
unit = "GB";
}
return value.toFixed(1) + " " + unit;
+ },
+ floatToFixed: function(value, fixed=2) {
+ if (value && value.toFixed) {
+ return parseFloat(value.toFixed(fixed));
+ }
+ return value;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-7258. Add Node and Rack Hints
to Opportunistic Scheduler. (Kartheek Muthyala via asuresh).
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
new file mode 100644
index 0000000..788b0b3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/scheduler/TestOpportunisticContainerAllocator.java
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RemoteNode;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class TestOpportunisticContainerAllocator {
+
+ private static final int GB = 1024;
+ private OpportunisticContainerAllocator allocator = null;
+ private OpportunisticContainerContext oppCntxt = null;
+
+ @Before
+ public void setup() {
+ SecurityUtil.setTokenServiceUseIp(false);
+ final MasterKey mKey = new MasterKey() {
+ @Override
+ public int getKeyId() {
+ return 1;
+ }
+ @Override
+ public void setKeyId(int keyId) {}
+ @Override
+ public ByteBuffer getBytes() {
+ return ByteBuffer.allocate(8);
+ }
+ @Override
+ public void setBytes(ByteBuffer bytes) {}
+ };
+ BaseContainerTokenSecretManager secMan =
+ new BaseContainerTokenSecretManager(new Configuration()) {
+ @Override
+ public MasterKey getCurrentKey() {
+ return mKey;
+ }
+
+ @Override
+ public byte[] createPassword(ContainerTokenIdentifier identifier) {
+ return new byte[]{1, 2};
+ }
+ };
+ allocator = new OpportunisticContainerAllocator(secMan);
+ oppCntxt = new OpportunisticContainerContext();
+ oppCntxt.getAppParams().setMinResource(Resource.newInstance(1024, 1));
+ oppCntxt.getAppParams().setIncrementResource(Resource.newInstance(512, 1));
+ oppCntxt.getAppParams().setMaxResource(Resource.newInstance(1024, 10));
+ }
+
+ @Test
+ public void testSimpleAllocation() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 1, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h1", 1234), "h1:1234", "/r1")));
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ Assert.assertEquals(1, containers.size());
+ Assert.assertEquals(0, oppCntxt.getOutstandingOpReqs().size());
+ }
+
+ @Test
+ public void testBlacklistRejection() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ Arrays.asList("h1", "h2"), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1),
+ "*", Resources.createResource(1 * GB), 1, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h1", 1234), "h1:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r2")));
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ Assert.assertEquals(0, containers.size());
+ Assert.assertEquals(1, oppCntxt.getOutstandingOpReqs().size());
+ }
+
+ @Test
+ public void testRoundRobinSimpleAllocation() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(3)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h1", 1234), "h1:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r1")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ System.out.println(containers);
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ Assert.assertTrue(allocatedHosts.contains("h1:1234"));
+ Assert.assertTrue(allocatedHosts.contains("h2:1234"));
+ Assert.assertTrue(allocatedHosts.contains("h3:1234"));
+ Assert.assertEquals(3, containers.size());
+ }
+
+ @Test
+ public void testNodeLocalAllocation() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h1", 1234), "h1:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r1")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ System.out.println(containers);
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ Assert.assertEquals(2, containers.size());
+ Assert.assertTrue(allocatedHosts.contains("h1:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h2:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h3:1234"));
+ }
+
+ @Test
+ public void testNodeLocalAllocationSameSchedKey() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .numContainers(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .numContainers(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .numContainers(2)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h1", 1234), "h1:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r1")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ System.out.println(containers);
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ Assert.assertEquals(2, containers.size());
+ Assert.assertTrue(allocatedHosts.contains("h1:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h2:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h3:1234"));
+ }
+
+ @Test
+ public void testSimpleRackLocalAllocation() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(1 * GB), 1, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "h1",
+ Resources.createResource(1 * GB), 1, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "/r1",
+ Resources.createResource(1 * GB), 1, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ Assert.assertTrue(allocatedHosts.contains("h2:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h3:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h4:1234"));
+ Assert.assertEquals(1, containers.size());
+ }
+
+ @Test
+ public void testRoundRobinRackLocalAllocation() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(1)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build(),
+ ResourceRequest.newBuilder().allocationRequestId(2)
+ .priority(Priority.newInstance(1))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h5", 1234), "h5:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ System.out.println(containers);
+ Assert.assertTrue(allocatedHosts.contains("h2:1234"));
+ Assert.assertTrue(allocatedHosts.contains("h5:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h3:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h4:1234"));
+ Assert.assertEquals(2, containers.size());
+ }
+
+ @Test
+ public void testRoundRobinRackLocalAllocationSameSchedKey() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "h1",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "/r1",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h5", 1234), "h5:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ Set<String> allocatedHosts = new HashSet<>();
+ for (Container c : containers) {
+ allocatedHosts.add(c.getNodeHttpAddress());
+ }
+ System.out.println(containers);
+ Assert.assertTrue(allocatedHosts.contains("h2:1234"));
+ Assert.assertTrue(allocatedHosts.contains("h5:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h3:1234"));
+ Assert.assertFalse(allocatedHosts.contains("h4:1234"));
+ Assert.assertEquals(2, containers.size());
+ }
+
+ @Test
+ public void testOffSwitchAllocationWhenNoNodeOrRack() throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "h6",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "/r3",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h5", 1234), "h5:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser");
+ System.out.println(containers);
+ Assert.assertEquals(2, containers.size());
+ }
+
+ @Test
+ public void testLotsOfContainersRackLocalAllocationSameSchedKey()
+ throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs =
+ Arrays.asList(
+ ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(1 * GB), 1000, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "h1",
+ Resources.createResource(1 * GB), 1000, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)),
+ ResourceRequest.newInstance(Priority.newInstance(1), "/r1",
+ Resources.createResource(1 * GB), 1000, true, null,
+ ExecutionTypeRequest.newInstance(
+ ExecutionType.OPPORTUNISTIC, true)));
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h5", 1234), "h5:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = new ArrayList<>();
+ for (int i = 0; i < 250; i++) {
+ containers.addAll(allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser"));
+ }
+ Assert.assertEquals(1000, containers.size());
+ }
+
+ @Test
+ public void testLotsOfContainersRackLocalAllocation()
+ throws Exception {
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ new ArrayList<>(), new ArrayList<>());
+ List<ResourceRequest> reqs = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ reqs.add(ResourceRequest.newBuilder().allocationRequestId(i + 1)
+ .priority(Priority.newInstance(1))
+ .resourceName("*")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ reqs.add(ResourceRequest.newBuilder().allocationRequestId(i + 1)
+ .priority(Priority.newInstance(1))
+ .resourceName("h1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ reqs.add(ResourceRequest.newBuilder().allocationRequestId(i + 1)
+ .priority(Priority.newInstance(1))
+ .resourceName("/r1")
+ .capability(Resources.createResource(1 * GB))
+ .relaxLocality(true)
+ .executionType(ExecutionType.OPPORTUNISTIC).build());
+ }
+ ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0L, 1), 1);
+
+ oppCntxt.updateNodeList(
+ Arrays.asList(
+ RemoteNode.newInstance(
+ NodeId.newInstance("h3", 1234), "h3:1234", "/r2"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h2", 1234), "h2:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h5", 1234), "h5:1234", "/r1"),
+ RemoteNode.newInstance(
+ NodeId.newInstance("h4", 1234), "h4:1234", "/r2")));
+
+ List<Container> containers = new ArrayList<>();
+ for (int i = 0; i < 25; i++) {
+ containers.addAll(allocator.allocateContainers(
+ blacklistRequest, reqs, appAttId, oppCntxt, 1L, "luser"));
+ }
+ Assert.assertEquals(100, containers.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index c3ed7d5..ce425df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -433,8 +433,12 @@ public class OpportunisticContainerAllocatorAMService
private RemoteNode convertToRemoteNode(NodeId nodeId) {
SchedulerNode node =
((AbstractYarnScheduler) rmContext.getScheduler()).getNode(nodeId);
- return node != null ? RemoteNode.newInstance(nodeId, node.getHttpAddress())
- : null;
+ if (node != null) {
+ RemoteNode rNode = RemoteNode.newInstance(nodeId, node.getHttpAddress());
+ rNode.setRackName(node.getRackName());
+ return rNode;
+ }
+ return null;
}
private static ApplicationAttemptId getAppAttemptId() throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 9b9eb3c..1af930f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -610,6 +610,8 @@ public class TestOpportunisticContainerAllocatorAMService {
.newInstance(ExecutionType.OPPORTUNISTIC, true))), null);
List<Container> allocatedContainers =
allocateResponse.getAllocatedContainers();
+ allocatedContainers.addAll(
+ am1.allocate(null, null).getAllocatedContainers());
Assert.assertEquals(2, allocatedContainers.size());
Container container = allocatedContainers.get(0);
// Start Container in NM
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-7046. Add closing logic to
configuration store
Posted by jh...@apache.org.
YARN-7046. Add closing logic to configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/674a7731
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/674a7731
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/674a7731
Branch: refs/heads/YARN-5734
Commit: 674a7731235f3d38acbed620cbbc91d1f3433184
Parents: 79fb0a5
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Fri Sep 22 10:18:27 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../scheduler/MutableConfigurationProvider.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 3 +
.../conf/LeveldbConfigurationStore.java | 44 ++--
.../conf/MutableCSConfigurationProvider.java | 6 +
.../capacity/conf/YarnConfigurationStore.java | 6 +
.../conf/ConfigurationStoreBaseTest.java | 2 +
.../conf/TestLeveldbConfigurationStore.java | 213 +++++++++++++++++++
7 files changed, 270 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index f8e8814..2b9b25a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import java.io.IOException;
+
/**
* Interface for allowing changing scheduler configurations.
*/
@@ -55,4 +57,10 @@ public interface MutableConfigurationProvider {
* @throws Exception if confirming mutation fails
*/
void confirmPendingMutation(boolean isValid) throws Exception;
+
+ /**
+ * Closes the configuration provider, releasing any required resources.
+ * @throws IOException on failure to close
+ */
+ void close() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 16b27c1..de95179 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -411,6 +411,9 @@ public class CapacityScheduler extends
writeLock.unlock();
}
+ if (isConfigurationMutable()) {
+ ((MutableConfigurationProvider) csConfProvider).close();
+ }
super.serviceStop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1b0eb9f..21de7a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -72,7 +73,8 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
private long maxLogs;
private Configuration conf;
private LogMutation pendingMutation;
- private static final Version CURRENT_VERSION_INFO = Version
+ @VisibleForTesting
+ protected static final Version CURRENT_VERSION_INFO = Version
.newInstance(0, 1);
private Timer compactionTimer;
private long compactionIntervalMsec;
@@ -82,7 +84,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
RMContext rmContext) throws IOException {
this.conf = config;
try {
- this.db = initDatabase(schedConf);
+ initDatabase(schedConf);
this.maxLogs = config.getLong(
YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
@@ -96,7 +98,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
}
- private DB initDatabase(Configuration config) throws Exception {
+ private void initDatabase(Configuration config) throws Exception {
Path storeRoot = createStorageDir();
Options options = new Options();
options.createIfMissing(false);
@@ -108,13 +110,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
if (key1Str.equals(key2Str)) {
return 0;
} else if (key1Str.equals(VERSION_KEY)) {
- return -1;
- } else if (key2Str.equals(VERSION_KEY)) {
return 1;
- } else if (key1Str.equals(LOG_KEY)) {
+ } else if (key2Str.equals(VERSION_KEY)) {
return -1;
- } else if (key2Str.equals(LOG_KEY)) {
+ } else if (key1Str.equals(LOG_KEY)) {
return 1;
+ } else if (key2Str.equals(LOG_KEY)) {
+ return -1;
}
return key1Str.compareTo(key2Str);
}
@@ -156,7 +158,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
throw e;
}
}
- return db;
}
private Path createStorageDir() throws IOException {
@@ -176,6 +177,13 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
@Override
+ public void close() throws IOException {
+ if (db != null) {
+ db.close();
+ }
+ }
+
+ @Override
public void logMutation(LogMutation logMutation) throws IOException {
LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
logs.add(logMutation);
@@ -212,8 +220,12 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return baos.toByteArray();
}
}
+
private LinkedList<LogMutation> deserLogMutations(byte[] mutations) throws
IOException {
+ if (mutations == null) {
+ return new LinkedList<>();
+ }
try (ObjectInput input = new ObjectInputStream(
new ByteArrayInputStream(mutations))) {
return (LinkedList<LogMutation>) input.readObject();
@@ -225,13 +237,16 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
@Override
public synchronized Configuration retrieve() {
DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_KEY));
+ itr.seekToFirst();
Configuration config = new Configuration(false);
- itr.next();
while (itr.hasNext()) {
Map.Entry<byte[], byte[]> entry = itr.next();
- config.set(new String(entry.getKey(), StandardCharsets.UTF_8),
- new String(entry.getValue(), StandardCharsets.UTF_8));
+ String key = new String(entry.getKey(), StandardCharsets.UTF_8);
+ String value = new String(entry.getValue(), StandardCharsets.UTF_8);
+ if (key.equals(LOG_KEY) || key.equals(VERSION_KEY)) {
+ break;
+ }
+ config.set(key, value);
}
return config;
}
@@ -268,6 +283,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return version;
}
+ @VisibleForTesting
+ protected LinkedList<LogMutation> getLogs() throws Exception {
+ return deserLogMutations(db.get(bytes(LOG_KEY)));
+ }
+
@Override
public void storeVersion() throws Exception {
String key = VERSION_KEY;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 70d1840..ccadf76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -92,6 +92,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
try {
confStore.initialize(config, schedConf, rmContext);
+ confStore.checkVersion();
} catch (Exception e) {
throw new IOException(e);
}
@@ -103,6 +104,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
aclMutationPolicy.init(config, rmContext);
}
+ @Override
+ public void close() throws IOException {
+ confStore.close();
+ }
+
@VisibleForTesting
public YarnConfigurationStore getConfStore() {
return confStore;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 1356535..7fb52fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -96,6 +96,12 @@ public abstract class YarnConfigurationStore {
RMContext rmContext) throws Exception;
/**
+ * Closes the configuration store, releasing any required resources.
+ * @throws IOException on failure to close
+ */
+ public void close() throws IOException {}
+
+ /**
* Logs the configuration change to backing store.
* @param logMutation configuration change to be persisted in write ahead log
* @throws IOException if logging fails
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
index bbe9570..8f3bc71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -71,6 +71,7 @@ public abstract class ConfigurationStoreBaseTest {
confStore.confirmMutation(false);
assertNull("Configuration should not be updated",
confStore.retrieve().get("keyUpdate2"));
+ confStore.close();
}
@Test
@@ -86,5 +87,6 @@ public abstract class ConfigurationStoreBaseTest {
confStore.logMutation(mutation);
confStore.confirmMutation(true);
assertNull(confStore.retrieve().get("key"));
+ confStore.close();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/674a7731/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
new file mode 100644
index 0000000..779208a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tests {@link LeveldbConfigurationStore}.
+ */
+public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestLeveldbConfigurationStore.class);
+ private static final File TEST_DIR = new File(
+ System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir")),
+ TestLeveldbConfigurationStore.class.getName());
+
+ private ResourceManager rm;
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ FileUtil.fullyDelete(TEST_DIR);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.LEVELDB_CONFIGURATION_STORE);
+ conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString());
+ }
+
+ @Test
+ public void testVersioning() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ assertEquals(LeveldbConfigurationStore.CURRENT_VERSION_INFO,
+ confStore.getConfStoreVersion());
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistConfiguration() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for old configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistUpdatedConfiguration() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", "val");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for updated configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ @Test
+ public void testMaxLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
+ confStore.initialize(conf, schedConf, rmContext);
+ LinkedList<YarnConfigurationStore.LogMutation> logs =
+ ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(0, logs.size());
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ confStore.confirmMutation(true);
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("key2", "val2");
+ mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+
+ // Next update should purge first update from logs.
+ Map<String, String> update3 = new HashMap<>();
+ update3.put("key3", "val3");
+ mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((LeveldbConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.close();
+ }
+
+ /**
+ * When restarting, RM should read from current state of store, including
+ * any updates from the previous RM instance.
+ * @throws Exception
+ */
+ @Test
+ public void testRestartReadsFromUpdatedStore() throws Exception {
+ ResourceManager rm1 = new MockRM(conf);
+ rm1.start();
+ assertNull(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+
+ // Update configuration on RM
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ schedConfUpdateInfo.getGlobalParams().put("key", "val");
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf, rm1.getRMContext());
+ assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+ confProvider.confirmPendingMutation(true);
+ assertEquals("val", ((MutableCSConfigurationProvider) confProvider)
+ .getConfStore().retrieve().get("key"));
+ // Next update is not persisted, it should not be recovered
+ schedConfUpdateInfo.getGlobalParams().put("key", "badVal");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.close();
+
+ // Start RM2 and verifies it starts with updated configuration
+ ResourceManager rm2 = new MockRM(conf);
+ rm2.start();
+ assertEquals("val", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve().get("key"));
+ assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("key"));
+ rm2.close();
+ }
+
+ @Override
+ public YarnConfigurationStore createConfStore() {
+ return new LeveldbConfigurationStore();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-6840. Implement zookeeper
based store for scheduler configuration updates. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
YARN-6840. Implement zookeeper based store for scheduler configuration updates. (Jonathan Hung via wangda)
Change-Id: I9debea674fe8c7e4109d4ca136965a1ea4c48bcc
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79fb0a58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79fb0a58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79fb0a58
Branch: refs/heads/YARN-5734
Commit: 79fb0a5841bd10e32b1bbbade1a09e38514b2afc
Parents: 33f1074
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Sep 18 09:53:42 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +-
.../src/main/resources/yarn-default.xml | 15 +-
.../server/resourcemanager/AdminService.java | 18 +-
.../server/resourcemanager/ResourceManager.java | 24 +-
.../RMStateVersionIncompatibleException.java | 2 +-
.../recovery/ZKRMStateStore.java | 5 +-
.../scheduler/MutableConfScheduler.java | 22 +-
.../scheduler/MutableConfigurationProvider.java | 36 ++-
.../scheduler/capacity/CapacityScheduler.java | 22 +-
.../conf/InMemoryConfigurationStore.java | 71 +++--
.../conf/LeveldbConfigurationStore.java | 168 +++++-----
.../conf/MutableCSConfigurationProvider.java | 148 +++++----
.../capacity/conf/YarnConfigurationStore.java | 132 ++++----
.../capacity/conf/ZKConfigurationStore.java | 235 ++++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 26 +-
.../conf/ConfigurationStoreBaseTest.java | 90 ++++++
.../conf/TestInMemoryConfigurationStore.java | 30 ++
.../TestMutableCSConfigurationProvider.java | 18 +-
.../conf/TestYarnConfigurationStore.java | 71 -----
.../capacity/conf/TestZKConfigurationStore.java | 312 +++++++++++++++++++
20 files changed, 1037 insertions(+), 422 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ea8652d..8809a7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -678,6 +678,7 @@ public class YarnConfiguration extends Configuration {
YARN_PREFIX + "scheduler.configuration.store.class";
public static final String MEMORY_CONFIGURATION_STORE = "memory";
public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ public static final String ZK_CONFIGURATION_STORE = "zk";
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
@@ -689,9 +690,16 @@ public class YarnConfiguration extends Configuration {
public static final long
DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
- public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
- YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
- public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ public static final String RM_SCHEDCONF_MAX_LOGS =
+ YARN_PREFIX + "scheduler.configuration.store.max-logs";
+ public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
+
+ /** Parent znode path under which ZKConfigurationStore will create znodes. */
+ public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX
+ + "scheduler.configuration.zk-store.parent-path";
+ public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH =
+ "/confstore";
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5afec1b..4516edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3412,11 +3412,20 @@
<property>
<description>
- The max number of configuration change log entries kept in LevelDB config
+ The max number of configuration change log entries kept in config
store, when yarn.scheduler.configuration.store.class is configured to be
- "leveldb". Default is 1000.
+ "leveldb" or "zk". Default is 1000 for either.
</description>
- <name>yarn.scheduler.configuration.leveldb-store.max-logs</name>
+ <name>yarn.scheduler.configuration.store.max-logs</name>
<value>1000</value>
</property>
+
+ <property>
+ <description>
+ ZK root node path for configuration store when using zookeeper-based
+ configuration store.
+ </description>
+ <name>yarn.scheduler.configuration.zk-store.parent-path</name>
+ <value>/confstore</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index fd9e849..6c0a854 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -387,9 +387,7 @@ public class AdminService extends CompositeService implements
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- ResourceScheduler scheduler = rm.getRMContext().getScheduler();
- if (scheduler instanceof MutableConfScheduler
- && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
+ if (isSchedulerMutable()) {
throw new IOException("Scheduler configuration is mutable. " +
operation + " is not allowed in this scenario.");
}
@@ -413,6 +411,12 @@ public class AdminService extends CompositeService implements
}
}
+ private boolean isSchedulerMutable() {
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ return (scheduler instanceof MutableConfScheduler
+ && ((MutableConfScheduler) scheduler).isConfigurationMutable());
+ }
+
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
throws YarnException, StandbyException {
@@ -721,6 +725,14 @@ public class AdminService extends CompositeService implements
void refreshAll() throws ServiceFailedException {
try {
checkAcls("refreshAll");
+ if (isSchedulerMutable()) {
+ try {
+ ((MutableConfScheduler) rm.getRMContext().getScheduler())
+ .getMutableConfProvider().reloadConfigurationFromStore();
+ } catch (Exception e) {
+ throw new IOException("Failed to refresh configuration:", e);
+ }
+ }
refreshQueues();
refreshNodes();
refreshSuperUserGroupsConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index fd95ee5..d9312f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -351,7 +351,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED);
if (curatorEnabled) {
- this.zkManager = createAndStartZKManager(conf);
+ this.zkManager = getAndStartZKManager(conf);
elector = new CuratorBasedElectorService(this);
} else {
elector = new ActiveStandbyElectorBasedElectorService(this);
@@ -360,13 +360,16 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
/**
- * Create and ZooKeeper Curator manager.
+ * Get ZooKeeper Curator manager, creating and starting if not exists.
* @param config Configuration for the ZooKeeper curator.
- * @return New ZooKeeper Curator manager.
+ * @return ZooKeeper Curator manager.
* @throws IOException If it cannot create the manager.
*/
- public ZKCuratorManager createAndStartZKManager(Configuration config)
- throws IOException {
+ public synchronized ZKCuratorManager getAndStartZKManager(Configuration
+ config) throws IOException {
+ if (this.zkManager != null) {
+ return zkManager;
+ }
ZKCuratorManager manager = new ZKCuratorManager(config);
// Get authentication
@@ -386,15 +389,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
manager.start(authInfos);
- return manager;
- }
-
- /**
- * Get the ZooKeeper Curator manager.
- * @return ZooKeeper Curator manager.
- */
- public ZKCuratorManager getZKManager() {
- return this.zkManager;
+ this.zkManager = manager;
+ return zkManager;
}
public CuratorFramework getCurator() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
index 135868f..d5fce36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateVersionIncompatibleException.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
/**
* This exception is thrown by ResourceManager if it's loading an incompatible
- * version of state from state store on recovery.
+ * version of storage on recovery.
*/
public class RMStateVersionIncompatibleException extends YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index ac67dcd..5bff77f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -327,10 +327,7 @@ public class ZKRMStateStore extends RMStateStore {
amrmTokenSecretManagerRoot =
getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT);
reservationRoot = getNodePath(zkRootNodePath, RESERVATION_SYSTEM_ROOT);
- zkManager = resourceManager.getZKManager();
- if (zkManager == null) {
- zkManager = resourceManager.createAndStartZKManager(conf);
- }
+ zkManager = resourceManager.getAndStartZKManager(conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 313bf6a..6f677fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
-
-import java.io.IOException;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,16 +26,6 @@ import java.io.IOException;
public interface MutableConfScheduler extends ResourceScheduler {
/**
- * Update the scheduler's configuration.
- * @param user Caller of this update
- * @param confUpdate configuration update
- * @throws IOException if scheduler could not be reinitialized
- * @throws YarnException if reservation system could not be reinitialized
- */
- void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException;
-
- /**
* Get the scheduler configuration.
* @return the scheduler configuration
*/
@@ -58,4 +43,11 @@ public interface MutableConfScheduler extends ResourceScheduler {
* @return whether scheduler configuration is mutable or not.
*/
boolean isConfigurationMutable();
+
+ /**
+ * Get scheduler's configuration provider, so other classes can directly
+ * call mutation APIs on configuration provider.
+ * @return scheduler's configuration provider
+ */
+ MutableConfigurationProvider getMutableConfProvider();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9baf1ad..f8e8814 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,30 +19,40 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
-import java.io.IOException;
-
/**
* Interface for allowing changing scheduler configurations.
*/
public interface MutableConfigurationProvider {
/**
- * Apply transactions which were not committed.
- * @throws IOException if recovery fails
+ * Get the acl mutation policy for this configuration provider.
+ * @return The acl mutation policy.
+ */
+ ConfigurationMutationACLPolicy getAclMutationPolicy();
+
+ /**
+ * Called when a new ResourceManager is starting/becomes active. Ensures
+ * configuration is up-to-date.
+ * @throws Exception if configuration could not be refreshed from store
*/
- void recoverConf() throws IOException;
+ void reloadConfigurationFromStore() throws Exception;
/**
- * Update the scheduler configuration with the provided key value pairs.
- * @param user User issuing the request
- * @param confUpdate Key-value pairs for configurations to be updated.
- * @throws IOException if scheduler could not be reinitialized
- * @throws YarnException if reservation system could not be reinitialized
+ * Log user's requested configuration mutation, and applies it in-memory.
+ * @param user User who requested the change
+ * @param confUpdate User's requested configuration change
+ * @throws Exception if logging the mutation fails
*/
- void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo
- confUpdate) throws IOException, YarnException;
+ void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo
+ confUpdate) throws Exception;
+ /**
+ * Confirm last logged mutation.
+ * @param isValid if the last logged mutation is applied to scheduler
+ * properly.
+ * @throws Exception if confirming mutation fails
+ */
+ void confirmPendingMutation(boolean isValid) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 51ee6a7..16b27c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -141,7 +141,6 @@ import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -393,9 +392,6 @@ public class CapacityScheduler extends
@Override
public void serviceStart() throws Exception {
startSchedulerThreads();
- if (this.csConfProvider instanceof MutableConfigurationProvider) {
- ((MutableConfigurationProvider) csConfProvider).recoverConf();
- }
super.serviceStart();
}
@@ -2619,19 +2615,15 @@ public class CapacityScheduler extends
}
@Override
- public void updateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
- if (isConfigurationMutable()) {
- ((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user, confUpdate);
- } else {
- throw new UnsupportedOperationException("Configured CS configuration " +
- "provider does not support updating configuration.");
- }
+ public boolean isConfigurationMutable() {
+ return csConfProvider instanceof MutableConfigurationProvider;
}
@Override
- public boolean isConfigurationMutable() {
- return csConfProvider instanceof MutableConfigurationProvider;
+ public MutableConfigurationProvider getMutableConfProvider() {
+ if (isConfigurationMutable()) {
+ return (MutableConfigurationProvider) csConfProvider;
+ }
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index c63734d..d69c236 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -19,8 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -28,48 +29,35 @@ import java.util.Map;
* A default implementation of {@link YarnConfigurationStore}. Doesn't offer
* persistent configuration storage, just stores the configuration in memory.
*/
-public class InMemoryConfigurationStore implements YarnConfigurationStore {
+public class InMemoryConfigurationStore extends YarnConfigurationStore {
private Configuration schedConf;
- private LinkedList<LogMutation> pendingMutations;
- private long pendingId;
+ private LogMutation pendingMutation;
@Override
- public void initialize(Configuration conf, Configuration schedConf) {
+ public void initialize(Configuration conf, Configuration schedConf,
+ RMContext rmContext) {
this.schedConf = schedConf;
- this.pendingMutations = new LinkedList<>();
- this.pendingId = 0;
}
@Override
- public synchronized long logMutation(LogMutation logMutation) {
- logMutation.setId(++pendingId);
- pendingMutations.add(logMutation);
- return pendingId;
+ public void logMutation(LogMutation logMutation) {
+ pendingMutation = logMutation;
}
@Override
- public synchronized boolean confirmMutation(long id, boolean isValid) {
- LogMutation mutation = pendingMutations.poll();
- // If confirmMutation is called out of order, discard mutations until id
- // is reached.
- while (mutation != null) {
- if (mutation.getId() == id) {
- if (isValid) {
- Map<String, String> mutations = mutation.getUpdates();
- for (Map.Entry<String, String> kv : mutations.entrySet()) {
- if (kv.getValue() == null) {
- schedConf.unset(kv.getKey());
- } else {
- schedConf.set(kv.getKey(), kv.getValue());
- }
- }
+ public void confirmMutation(boolean isValid) {
+ if (isValid) {
+ for (Map.Entry<String, String> kv : pendingMutation.getUpdates()
+ .entrySet()) {
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
}
- return true;
}
- mutation = pendingMutations.poll();
}
- return false;
+ pendingMutation = null;
}
@Override
@@ -78,13 +66,30 @@ public class InMemoryConfigurationStore implements YarnConfigurationStore {
}
@Override
- public synchronized List<LogMutation> getPendingMutations() {
- return new LinkedList<>(pendingMutations);
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
}
@Override
- public List<LogMutation> getConfirmedConfHistory(long fromId) {
- // Unimplemented.
+ public Version getConfStoreVersion() throws Exception {
+ // Does nothing.
return null;
}
+
+ @Override
+ public void storeVersion() throws Exception {
+ // Does nothing.
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ // Does nothing.
+ return null;
+ }
+
+ @Override
+ public void checkVersion() {
+ // Does nothing. (Version is always compatible since it's in memory)
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 1280fab..1b0eb9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -26,6 +26,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
@@ -55,58 +59,32 @@ import static org.fusesource.leveldbjni.JniDBFactory.bytes;
/**
* A LevelDB implementation of {@link YarnConfigurationStore}.
*/
-public class LeveldbConfigurationStore implements YarnConfigurationStore {
+public class LeveldbConfigurationStore extends YarnConfigurationStore {
public static final Log LOG =
LogFactory.getLog(LeveldbConfigurationStore.class);
private static final String DB_NAME = "yarn-conf-store";
- private static final String LOG_PREFIX = "log.";
- private static final String LOG_COMMITTED_TXN = "committedTxn";
+ private static final String LOG_KEY = "log";
+ private static final String VERSION_KEY = "version";
private DB db;
- // Txnid for the last transaction logged to the store.
- private long txnId = 0;
- private long minTxn = 0;
private long maxLogs;
private Configuration conf;
- private LinkedList<LogMutation> pendingMutations = new LinkedList<>();
+ private LogMutation pendingMutation;
+ private static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(0, 1);
private Timer compactionTimer;
private long compactionIntervalMsec;
@Override
- public void initialize(Configuration config, Configuration schedConf)
- throws IOException {
+ public void initialize(Configuration config, Configuration schedConf,
+ RMContext rmContext) throws IOException {
this.conf = config;
try {
this.db = initDatabase(schedConf);
- this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)),
- StandardCharsets.UTF_8));
- DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_PREFIX + txnId));
- // Seek to first uncommitted log
- itr.next();
- while (itr.hasNext()) {
- Map.Entry<byte[], byte[]> entry = itr.next();
- if (!new String(entry.getKey(), StandardCharsets.UTF_8)
- .startsWith(LOG_PREFIX)) {
- break;
- }
- pendingMutations.add(deserLogMutation(entry.getValue()));
- txnId++;
- }
- // Get the earliest txnId stored in logs
- itr.seekToFirst();
- if (itr.hasNext()) {
- Map.Entry<byte[], byte[]> entry = itr.next();
- byte[] key = entry.getKey();
- String logId = new String(key, StandardCharsets.UTF_8);
- if (logId.startsWith(LOG_PREFIX)) {
- minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1));
- }
- }
this.maxLogs = config.getLong(
- YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS,
+ YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
this.compactionIntervalMsec = config.getLong(
YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS,
@@ -127,33 +105,23 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
public int compare(byte[] key1, byte[] key2) {
String key1Str = new String(key1, StandardCharsets.UTF_8);
String key2Str = new String(key2, StandardCharsets.UTF_8);
- int key1Txn = Integer.MAX_VALUE;
- int key2Txn = Integer.MAX_VALUE;
- if (key1Str.startsWith(LOG_PREFIX)) {
- key1Txn = Integer.parseInt(key1Str.substring(
- key1Str.indexOf('.') + 1));
- }
- if (key2Str.startsWith(LOG_PREFIX)) {
- key2Txn = Integer.parseInt(key2Str.substring(
- key2Str.indexOf('.') + 1));
- }
- // TODO txnId could overflow, in theory
- if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) {
- if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) {
- return 0;
- } else if (key1Str.equals(LOG_COMMITTED_TXN)) {
- return -1;
- } else if (key2Str.equals(LOG_COMMITTED_TXN)) {
- return 1;
- }
- return key1Str.compareTo(key2Str);
+ if (key1Str.equals(key2Str)) {
+ return 0;
+ } else if (key1Str.equals(VERSION_KEY)) {
+ return -1;
+ } else if (key2Str.equals(VERSION_KEY)) {
+ return 1;
+ } else if (key1Str.equals(LOG_KEY)) {
+ return -1;
+ } else if (key2Str.equals(LOG_KEY)) {
+ return 1;
}
- return key1Txn - key2Txn;
+ return key1Str.compareTo(key2Str);
}
@Override
public String name() {
- return "logComparator";
+ return "keyComparator";
}
public byte[] findShortestSeparator(byte[] start, byte[] limit) {
@@ -164,6 +132,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
return key;
}
});
+
LOG.info("Using conf database at " + storeRoot);
File dbfile = new File(storeRoot.toString());
try {
@@ -179,7 +148,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
for (Map.Entry<String, String> kv : config) {
initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
}
- initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0"));
db.write(initBatch);
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
@@ -208,28 +176,22 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
@Override
- public synchronized long logMutation(LogMutation logMutation)
- throws IOException {
- logMutation.setId(++txnId);
- WriteBatch logBatch = db.createWriteBatch();
- logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation));
- if (txnId - minTxn >= maxLogs) {
- logBatch.delete(bytes(LOG_PREFIX + minTxn));
- minTxn++;
+ public void logMutation(LogMutation logMutation) throws IOException {
+ LinkedList<LogMutation> logs = deserLogMutations(db.get(bytes(LOG_KEY)));
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.removeFirst();
}
- db.write(logBatch);
- pendingMutations.add(logMutation);
- return txnId;
+ db.put(bytes(LOG_KEY), serLogMutations(logs));
+ pendingMutation = logMutation;
}
@Override
- public synchronized boolean confirmMutation(long id, boolean isValid)
- throws IOException {
+ public void confirmMutation(boolean isValid) throws IOException {
WriteBatch updateBatch = db.createWriteBatch();
if (isValid) {
- LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id)));
for (Map.Entry<String, String> changes :
- mutation.getUpdates().entrySet()) {
+ pendingMutation.getUpdates().entrySet()) {
if (changes.getValue() == null || changes.getValue().isEmpty()) {
updateBatch.delete(bytes(changes.getKey()));
} else {
@@ -237,28 +199,24 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
}
}
- updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id)));
db.write(updateBatch);
- // Assumes logMutation and confirmMutation are done in the same
- // synchronized method. For example,
- // {@link MutableCSConfigurationProvider#mutateConfiguration(
- // UserGroupInformation user, SchedConfUpdateInfo confUpdate)}
- pendingMutations.removeFirst();
- return true;
+ pendingMutation = null;
}
- private byte[] serLogMutation(LogMutation mutation) throws IOException {
+ private byte[] serLogMutations(LinkedList<LogMutation> mutations) throws
+ IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ObjectOutput oos = new ObjectOutputStream(baos)) {
- oos.writeObject(mutation);
+ oos.writeObject(mutations);
oos.flush();
return baos.toByteArray();
}
}
- private LogMutation deserLogMutation(byte[] mutation) throws IOException {
+ private LinkedList<LogMutation> deserLogMutations(byte[] mutations) throws
+ IOException {
try (ObjectInput input = new ObjectInputStream(
- new ByteArrayInputStream(mutation))) {
- return (LogMutation) input.readObject();
+ new ByteArrayInputStream(mutations))) {
+ return (LinkedList<LogMutation>) input.readObject();
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
@@ -267,7 +225,7 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
@Override
public synchronized Configuration retrieve() {
DBIterator itr = db.iterator();
- itr.seek(bytes(LOG_COMMITTED_TXN));
+ itr.seek(bytes(LOG_KEY));
Configuration config = new Configuration(false);
itr.next();
while (itr.hasNext()) {
@@ -279,11 +237,6 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
@Override
- public List<LogMutation> getPendingMutations() {
- return new LinkedList<>(pendingMutations);
- }
-
- @Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented
}
@@ -299,6 +252,39 @@ public class LeveldbConfigurationStore implements YarnConfigurationStore {
}
}
+ // TODO: following is taken from LeveldbRMStateStore
+ @Override
+ public Version getConfStoreVersion() throws Exception {
+ Version version = null;
+ try {
+ byte[] data = db.get(bytes(VERSION_KEY));
+ if (data != null) {
+ version = new VersionPBImpl(YarnServerCommonProtos.VersionProto
+ .parseFrom(data));
+ }
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ return version;
+ }
+
+ @Override
+ public void storeVersion() throws Exception {
+ String key = VERSION_KEY;
+ byte[] data = ((VersionPBImpl) CURRENT_VERSION_INFO).getProto()
+ .toByteArray();
+ try {
+ db.put(bytes(key), data);
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
private class CompactionTimerTask extends TimerTask {
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index d03b2e2..70d1840 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,20 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -56,6 +53,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
LogFactory.getLog(MutableCSConfigurationProvider.class);
private Configuration schedConf;
+ private Configuration oldConf;
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
@@ -76,6 +74,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
this.confStore = new LeveldbConfigurationStore();
break;
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ this.confStore = new ZKConfigurationStore();
+ break;
default:
this.confStore = YarnConfigurationStoreFactory.getStore(config);
break;
@@ -89,7 +90,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
for (Map.Entry<String, String> kv : initialSchedConf) {
schedConf.set(kv.getKey(), kv.getValue());
}
- confStore.initialize(config, schedConf);
+ try {
+ confStore.initialize(config, schedConf, rmContext);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
// After initializing confStore, the store may already have an existing
// configuration. Use this one.
schedConf = confStore.retrieve();
@@ -98,6 +103,11 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
aclMutationPolicy.init(config, rmContext);
}
+ @VisibleForTesting
+ public YarnConfigurationStore getConfStore() {
+ return confStore;
+ }
+
@Override
public CapacitySchedulerConfiguration loadConfiguration(Configuration
configuration) throws IOException {
@@ -107,16 +117,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public synchronized void mutateConfiguration(UserGroupInformation user,
- SchedConfUpdateInfo confUpdate) throws IOException, YarnException {
- if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
- throw new AccessControlException("User is not admin of all modified" +
- " queues.");
- }
- Configuration oldConf = new Configuration(schedConf);
+ public ConfigurationMutationACLPolicy getAclMutationPolicy() {
+ return aclMutationPolicy;
+ }
+
+ @Override
+ public void logAndApplyMutation(UserGroupInformation user,
+ SchedConfUpdateInfo confUpdate) throws Exception {
+ oldConf = new Configuration(schedConf);
Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
- long id = confStore.logMutation(log);
+ confStore.logMutation(log);
for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
@@ -124,47 +135,33 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
}
- try {
- rmContext.getRMAdminService().refreshQueues();
- } catch (IOException | YarnException e) {
+ }
+
+ @Override
+ public void confirmPendingMutation(boolean isValid) throws Exception {
+ confStore.confirmMutation(isValid);
+ if (!isValid) {
schedConf = oldConf;
- confStore.confirmMutation(id, false);
- throw e;
}
- confStore.confirmMutation(id, true);
}
@Override
- public void recoverConf() throws IOException {
- List<LogMutation> uncommittedLogs = confStore.getPendingMutations();
- Configuration oldConf = new Configuration(schedConf);
- for (LogMutation mutation : uncommittedLogs) {
- for (Map.Entry<String, String> kv : mutation.getUpdates().entrySet()) {
- if (kv.getValue() == null) {
- schedConf.unset(kv.getKey());
- } else {
- schedConf.set(kv.getKey(), kv.getValue());
- }
- }
- try {
- rmContext.getScheduler().reinitialize(schedConf, rmContext);
- } catch (IOException e) {
- schedConf = oldConf;
- confStore.confirmMutation(mutation.getId(), false);
- LOG.info("Configuration mutation " + mutation.getId()
- + " was rejected", e);
- continue;
- }
- confStore.confirmMutation(mutation.getId(), true);
- LOG.info("Configuration mutation " + mutation.getId()+ " was accepted");
- }
+ public void reloadConfigurationFromStore() throws Exception {
+ schedConf = confStore.retrieve();
+ }
+
+ private List<String> getSiblingQueues(String queuePath, Configuration conf) {
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
+ parentQueue + CapacitySchedulerConfiguration.DOT +
+ CapacitySchedulerConfiguration.QUEUES;
+ return new ArrayList<>(conf.getStringCollection(childQueuesKey));
}
private Map<String, String> constructKeyValueConfUpdate(
SchedConfUpdateInfo mutationInfo) throws IOException {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ new CapacitySchedulerConfiguration(schedConf, false);
Map<String, String> confUpdate = new HashMap<>();
for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
removeQueue(queueToRemove, proposedConf, confUpdate);
@@ -188,40 +185,35 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
if (queueToRemove == null) {
return;
} else {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
String queueName = queueToRemove.substring(
queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
+ if (queueToRemove.lastIndexOf('.') == -1) {
throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX
- + parentQueuePath + CapacitySchedulerConfiguration.DOT
- + CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
} else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
+ List<String> siblingQueues = getSiblingQueues(queueToRemove,
+ proposedConf);
+ if (!siblingQueues.contains(queueName)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ }
+ siblingQueues.remove(queueName);
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(
+ new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (siblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
}
}
}
@@ -232,13 +224,13 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
if (addInfo == null) {
return;
} else {
- CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
String queuePath = addInfo.getQueue();
String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
+ if (queuePath.lastIndexOf('.') == -1) {
throw new IOException("Can't add invalid queue " + queuePath);
+ } else if (getSiblingQueues(queuePath, proposedConf).contains(
+ queueName)) {
+ throw new IOException("Can't add existing queue " + queuePath);
}
String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
String[] siblings = proposedConf.getQueues(parentQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 065c877..1356535 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateVersionIncompatibleException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import java.io.IOException;
@@ -39,36 +44,26 @@ import java.util.Map;
* {@code getPendingMutations}, and replay/confirm them via
* {@code confirmMutation} as in the normal case.
*/
-public interface YarnConfigurationStore {
+public abstract class YarnConfigurationStore {
+ public static final Log LOG =
+ LogFactory.getLog(YarnConfigurationStore.class);
/**
* LogMutation encapsulates the fields needed for configuration mutation
* audit logging and recovery.
*/
- class LogMutation implements Serializable {
+ static class LogMutation implements Serializable {
private Map<String, String> updates;
private String user;
- private long id;
/**
- * Create log mutation prior to logging.
+ * Create log mutation.
* @param updates key-value configuration updates
* @param user user who requested configuration change
*/
- public LogMutation(Map<String, String> updates, String user) {
- this(updates, user, 0);
- }
-
- /**
- * Create log mutation for recovery.
- * @param updates key-value configuration updates
- * @param user user who requested configuration change
- * @param id transaction id of configuration change
- */
- LogMutation(Map<String, String> updates, String user, long id) {
+ LogMutation(Map<String, String> updates, String user) {
this.updates = updates;
this.user = user;
- this.id = id;
}
/**
@@ -86,75 +81,92 @@ public interface YarnConfigurationStore {
public String getUser() {
return user;
}
-
- /**
- * Get transaction id of this configuration change.
- * @return transaction id
- */
- public long getId() {
- return id;
- }
-
- /**
- * Set transaction id of this configuration change.
- * @param id transaction id
- */
- public void setId(long id) {
- this.id = id;
- }
}
/**
- * Initialize the configuration store.
+ * Initialize the configuration store, with schedConf as the initial
+ * scheduler configuration. If a persisted store already exists, use the
+ * scheduler configuration stored there, and ignore schedConf.
* @param conf configuration to initialize store with
- * @param schedConf Initial key-value configuration to persist
+ * @param schedConf Initial key-value scheduler configuration to persist.
+ * @param rmContext RMContext for this configuration store
* @throws IOException if initialization fails
*/
- void initialize(Configuration conf, Configuration schedConf)
- throws IOException;
+ public abstract void initialize(Configuration conf, Configuration schedConf,
+ RMContext rmContext) throws Exception;
/**
- * Logs the configuration change to backing store. Generates an id associated
- * with this mutation, sets it in {@code logMutation}, and returns it.
+ * Logs the configuration change to backing store.
* @param logMutation configuration change to be persisted in write ahead log
- * @return id which configuration store associates with this mutation
* @throws IOException if logging fails
*/
- long logMutation(LogMutation logMutation) throws IOException;
+ public abstract void logMutation(LogMutation logMutation) throws Exception;
/**
* Should be called after {@code logMutation}. Gets the pending mutation
- * associated with {@code id} and marks the mutation as persisted (no longer
- * pending). If isValid is true, merge the mutation with the persisted
+ * last logged by {@code logMutation} and marks the mutation as persisted (no
+ * longer pending). If isValid is true, merge the mutation with the persisted
* configuration.
- *
- * If {@code confirmMutation} is called with ids in a different order than
- * was returned by {@code logMutation}, the result is implementation
- * dependent.
- * @param id id of mutation to be confirmed
- * @param isValid if true, update persisted configuration with mutation
- * associated with {@code id}.
- * @return true on success
- * @throws IOException if mutation confirmation fails
+ * @param isValid if true, update persisted configuration with pending
+ * mutation.
+ * @throws Exception if mutation confirmation fails
*/
- boolean confirmMutation(long id, boolean isValid) throws IOException;
+ public abstract void confirmMutation(boolean isValid) throws Exception;
/**
* Retrieve the persisted configuration.
* @return configuration as key-value
*/
- Configuration retrieve();
-
- /**
- * Get the list of pending mutations, in the order they were logged.
- * @return list of mutations
- */
- List<LogMutation> getPendingMutations();
+ public abstract Configuration retrieve();
/**
* Get a list of confirmed configuration mutations starting from a given id.
* @param fromId id from which to start getting mutations, inclusive
* @return list of configuration mutations
*/
- List<LogMutation> getConfirmedConfHistory(long fromId);
+ public abstract List<LogMutation> getConfirmedConfHistory(long fromId);
+
+ /**
+ * Get schema version of persisted conf store, for detecting compatibility
+ * issues when changing conf store schema.
+ * @return Schema version currently used by the persisted configuration store.
+ * @throws Exception On version fetch failure
+ */
+ protected abstract Version getConfStoreVersion() throws Exception;
+
+ /**
+ * Persist the hard-coded schema version to the conf store.
+ * @throws Exception On storage failure
+ */
+ protected abstract void storeVersion() throws Exception;
+
+ /**
+ * Get the hard-coded schema version, for comparison against the schema
+ * version currently persisted.
+ * @return Current hard-coded schema version
+ */
+ protected abstract Version getCurrentVersion();
+
+ public void checkVersion() throws Exception {
+ // TODO this was taken from RMStateStore. Should probably refactor
+ Version loadedVersion = getConfStoreVersion();
+ LOG.info("Loaded configuration store version info " + loadedVersion);
+ if (loadedVersion != null && loadedVersion.equals(getCurrentVersion())) {
+ return;
+ }
+ // if there is no version info, treat it as CURRENT_VERSION_INFO;
+ if (loadedVersion == null) {
+ loadedVersion = getCurrentVersion();
+ }
+ if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+ LOG.info("Storing configuration store version info "
+ + getCurrentVersion());
+ storeVersion();
+ } else {
+ throw new RMStateVersionIncompatibleException(
+ "Expecting configuration store version " + getCurrentVersion()
+ + ", but loading version " + loadedVersion);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
new file mode 100644
index 0000000..a0bba8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A Zookeeper-based implementation of {@link YarnConfigurationStore}.
+ */
+public class ZKConfigurationStore extends YarnConfigurationStore {
+
+ public static final Log LOG =
+ LogFactory.getLog(ZKConfigurationStore.class);
+
+ private long maxLogs;
+
+ @VisibleForTesting
+ protected static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(0, 1);
+ private Configuration conf;
+ private LogMutation pendingMutation;
+
+ private String znodeParentPath;
+
+ private static final String ZK_VERSION_PATH = "VERSION";
+ private static final String LOGS_PATH = "LOGS";
+ private static final String CONF_STORE_PATH = "CONF_STORE";
+ private static final String FENCING_PATH = "FENCING";
+
+ private String zkVersionPath;
+ private String logsPath;
+ private String confStorePath;
+ private String fencingNodePath;
+
+ @VisibleForTesting
+ protected ZKCuratorManager zkManager;
+ private List<ACL> zkAcl;
+
+ @Override
+ public void initialize(Configuration config, Configuration schedConf,
+ RMContext rmContext) throws Exception {
+ this.conf = config;
+ this.maxLogs = conf.getLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS);
+ this.znodeParentPath =
+ conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
+ this.zkManager = rmContext.getResourceManager().getAndStartZKManager(conf);
+ this.zkAcl = ZKCuratorManager.getZKAcls(conf);
+
+ this.zkVersionPath = getNodePath(znodeParentPath, ZK_VERSION_PATH);
+ this.logsPath = getNodePath(znodeParentPath, LOGS_PATH);
+ this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH);
+ this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH);
+
+ zkManager.createRootDirRecursively(znodeParentPath);
+ zkManager.delete(fencingNodePath);
+
+ if (!zkManager.exists(logsPath)) {
+ zkManager.create(logsPath);
+ zkManager.setData(logsPath,
+ serializeObject(new LinkedList<LogMutation>()), -1);
+ }
+
+ if (!zkManager.exists(confStorePath)) {
+ zkManager.create(confStorePath);
+ HashMap<String, String> mapSchedConf = new HashMap<>();
+ for (Map.Entry<String, String> entry : schedConf) {
+ mapSchedConf.put(entry.getKey(), entry.getValue());
+ }
+ zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1);
+ }
+ }
+
+ @VisibleForTesting
+ protected LinkedList<LogMutation> getLogs() throws Exception {
+ return (LinkedList<LogMutation>)
+ deserializeObject(zkManager.getData(logsPath));
+ }
+
+ // TODO: following version-related code is taken from ZKRMStateStore
+ @Override
+ public Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ @Override
+ public Version getConfStoreVersion() throws Exception {
+ if (zkManager.exists(zkVersionPath)) {
+ byte[] data = zkManager.getData(zkVersionPath);
+ return new VersionPBImpl(YarnServerCommonProtos.VersionProto
+ .parseFrom(data));
+ }
+
+ return null;
+ }
+
+ @Override
+ public synchronized void storeVersion() throws Exception {
+ byte[] data =
+ ((VersionPBImpl) CURRENT_VERSION_INFO).getProto().toByteArray();
+
+ if (zkManager.exists(zkVersionPath)) {
+ zkManager.safeSetData(zkVersionPath, data, -1, zkAcl, fencingNodePath);
+ } else {
+ zkManager.safeCreate(zkVersionPath, data, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
+ }
+ }
+
+ @Override
+ public void logMutation(LogMutation logMutation) throws Exception {
+ byte[] storedLogs = zkManager.getData(logsPath);
+ LinkedList<LogMutation> logs = new LinkedList<>();
+ if (storedLogs != null) {
+ logs = (LinkedList<LogMutation>) deserializeObject(storedLogs);
+ }
+ logs.add(logMutation);
+ if (logs.size() > maxLogs) {
+ logs.remove(logs.removeFirst());
+ }
+ zkManager.safeSetData(logsPath, serializeObject(logs), -1, zkAcl,
+ fencingNodePath);
+ pendingMutation = logMutation;
+ }
+
+ @Override
+ public void confirmMutation(boolean isValid)
+ throws Exception {
+ if (isValid) {
+ Configuration storedConfigs = retrieve();
+ Map<String, String> mapConf = new HashMap<>();
+ for (Map.Entry<String, String> storedConf : storedConfigs) {
+ mapConf.put(storedConf.getKey(), storedConf.getValue());
+ }
+ for (Map.Entry<String, String> confChange :
+ pendingMutation.getUpdates().entrySet()) {
+ if (confChange.getValue() == null || confChange.getValue().isEmpty()) {
+ mapConf.remove(confChange.getKey());
+ } else {
+ mapConf.put(confChange.getKey(), confChange.getValue());
+ }
+ }
+ zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1,
+ zkAcl, fencingNodePath);
+ }
+ pendingMutation = null;
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ byte[] serializedSchedConf;
+ try {
+ serializedSchedConf = zkManager.getData(confStorePath);
+ } catch (Exception e) {
+ LOG.error("Failed to retrieve configuration from zookeeper store", e);
+ return null;
+ }
+ try {
+ Map<String, String> map =
+ (HashMap<String, String>) deserializeObject(serializedSchedConf);
+ Configuration c = new Configuration();
+ for (Map.Entry<String, String> e : map.entrySet()) {
+ c.set(e.getKey(), e.getValue());
+ }
+ return c;
+ } catch (Exception e) {
+ LOG.error("Exception while deserializing scheduler configuration " +
+ "from store", e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ return null; // unimplemented
+ }
+
+ private static String getNodePath(String root, String nodeName) {
+ return ZKCuratorManager.getNodePath(root, nodeName);
+ }
+
+ private static byte[] serializeObject(Object o) throws Exception {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);) {
+ oos.writeObject(o);
+ oos.flush();
+ baos.flush();
+ return baos.toByteArray();
+ }
+ }
+
+ private static Object deserializeObject(byte[] bytes) throws Exception {
+ try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ ObjectInputStream ois = new ObjectInputStream(bais);) {
+ return ois.readObject();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 1da4e65..d264c10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -2464,7 +2465,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public Response updateSchedulerConfiguration(SchedConfUpdateInfo
+ public synchronized Response updateSchedulerConfiguration(SchedConfUpdateInfo
mutationInfo, @Context HttpServletRequest hsr)
throws AuthorizationException, InterruptedException {
init();
@@ -2479,17 +2480,32 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
ResourceScheduler scheduler = rm.getResourceScheduler();
- if (scheduler instanceof MutableConfScheduler) {
+ if (scheduler instanceof MutableConfScheduler && ((MutableConfScheduler)
+ scheduler).isConfigurationMutable()) {
try {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws IOException, YarnException {
- ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
- mutationInfo);
+ public Void run() throws Exception {
+ MutableConfigurationProvider provider = ((MutableConfScheduler)
+ scheduler).getMutableConfProvider();
+ if (!provider.getAclMutationPolicy().isMutationAllowed(callerUGI,
+ mutationInfo)) {
+ throw new org.apache.hadoop.security.AccessControlException("User"
+ + " is not admin of all modified queues.");
+ }
+ provider.logAndApplyMutation(callerUGI, mutationInfo);
+ try {
+ rm.getRMContext().getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
+ provider.confirmPendingMutation(false);
+ throw e;
+ }
+ provider.confirmPendingMutation(true);
return null;
}
});
} catch (IOException e) {
+ LOG.error("Exception thrown when modifying configuration.", e);
return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
new file mode 100644
index 0000000..bbe9570
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Base class for {@link YarnConfigurationStore} implementations.
+ */
+public abstract class ConfigurationStoreBaseTest {
+
+ protected YarnConfigurationStore confStore = createConfStore();
+
+ protected abstract YarnConfigurationStore createConfStore();
+
+ protected Configuration conf;
+ protected Configuration schedConf;
+ protected RMContext rmContext;
+
+ protected static final String TEST_USER = "testUser";
+
+ @Before
+ public void setUp() throws Exception {
+ this.conf = new Configuration();
+ this.schedConf = new Configuration(false);
+ }
+
+ @Test
+ public void testConfigurationUpdate() throws Exception {
+ schedConf.set("key1", "val1");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val1", confStore.retrieve().get("key1"));
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("keyUpdate1", "valUpdate1");
+ YarnConfigurationStore.LogMutation mutation1 =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation1);
+ confStore.confirmMutation(true);
+ assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("keyUpdate2", "valUpdate2");
+ YarnConfigurationStore.LogMutation mutation2 =
+ new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation2);
+ confStore.confirmMutation(false);
+ assertNull("Configuration should not be updated",
+ confStore.retrieve().get("keyUpdate2"));
+ }
+
+ @Test
+ public void testNullConfigurationUpdate() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", null);
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertNull(confStore.retrieve().get("key"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
new file mode 100644
index 0000000..c40d16a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestInMemoryConfigurationStore.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+/**
+ * Tests {@link InMemoryConfigurationStore}.
+ */
+public class TestInMemoryConfigurationStore extends ConfigurationStoreBaseTest {
+
+ @Override
+ protected YarnConfigurationStore createConfStore() {
+ return new InMemoryConfigurationStore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 635a184..9b080cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -30,14 +29,11 @@ import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -82,25 +78,21 @@ public class TestMutableCSConfigurationProvider {
}
@Test
- public void testInMemoryBackedProvider() throws IOException, YarnException {
+ public void testInMemoryBackedProvider() throws Exception {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
- doNothing().when(adminService).refreshQueues();
- confProvider.mutateConfiguration(TEST_USER, goodUpdate);
+ confProvider.logAndApplyMutation(TEST_USER, goodUpdate);
+ confProvider.confirmPendingMutation(true);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
- doThrow(new IOException()).when(adminService).refreshQueues();
- try {
- confProvider.mutateConfiguration(TEST_USER, badUpdate);
- } catch (IOException e) {
- // Expected exception.
- }
+ confProvider.logAndApplyMutation(TEST_USER, badUpdate);
+ confProvider.confirmPendingMutation(false);
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
deleted file mode 100644
index 631ce65..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class TestYarnConfigurationStore {
-
- private YarnConfigurationStore confStore;
- private Configuration schedConf;
-
- private static final String testUser = "testUser";
-
- @Before
- public void setUp() {
- schedConf = new Configuration(false);
- schedConf.set("key1", "val1");
- }
-
- @Test
- public void testInMemoryConfigurationStore() throws IOException {
- confStore = new InMemoryConfigurationStore();
- confStore.initialize(new Configuration(), schedConf);
- assertEquals("val1", confStore.retrieve().get("key1"));
-
- Map<String, String> update1 = new HashMap<>();
- update1.put("keyUpdate1", "valUpdate1");
- LogMutation mutation1 = new LogMutation(update1, testUser);
- long id = confStore.logMutation(mutation1);
- assertEquals(1, confStore.getPendingMutations().size());
- confStore.confirmMutation(id, true);
- assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
- assertEquals(0, confStore.getPendingMutations().size());
-
- Map<String, String> update2 = new HashMap<>();
- update2.put("keyUpdate2", "valUpdate2");
- LogMutation mutation2 = new LogMutation(update2, testUser);
- id = confStore.logMutation(mutation2);
- assertEquals(1, confStore.getPendingMutations().size());
- confStore.confirmMutation(id, false);
- assertNull("Configuration should not be updated",
- confStore.retrieve().get("keyUpdate2"));
- assertEquals(0, confStore.getPendingMutations().size());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-5947: Create
LeveldbConfigurationStore class using Leveldb as backing store. Contributed
by Jonathan Hung
Posted by jh...@apache.org.
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f127c549
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f127c549
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f127c549
Branch: refs/heads/YARN-5734
Commit: f127c549c3973db4d6d3df21d5e562f9e459ca32
Parents: 376ab6b
Author: Xuan <xg...@apache.org>
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../src/main/resources/yarn-default.xml | 29 ++
.../scheduler/MutableConfigurationProvider.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 3 +
.../conf/LeveldbConfigurationStore.java | 314 +++++++++++++++++++
.../conf/MutableCSConfigurationProvider.java | 38 ++-
.../capacity/conf/YarnConfigurationStore.java | 14 +-
.../conf/TestYarnConfigurationStore.java | 3 +-
8 files changed, 414 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
YARN_PREFIX + "scheduler.configuration.store.class";
public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+ + "scheduler.configuration.leveldb-store.path";
+
+ public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+ YARN_PREFIX
+ + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+ public static final long
+ DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+ public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+ YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+ public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
</property>
+ <property>
+ <description>
+ The storage path for LevelDB implementation of configuration store,
+ when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb".
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.path</name>
+ <value>${hadoop.tmp.dir}/yarn/system/confstore</value>
+ </property>
+
+ <property>
+ <description>
+ The compaction interval for LevelDB configuration store in secs,
+ when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb". Default is one day.
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.compaction-interval-secs</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <description>
+ The max number of configuration change log entries kept in LevelDB config
+ store, when yarn.scheduler.configuration.store.class is configured to be
+ "leveldb". Default is 1000.
+ </description>
+ <name>yarn.scheduler.configuration.leveldb-store.max-logs</name>
+ <value>1000</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 86be7c3..1f13467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -29,6 +29,12 @@ import java.io.IOException;
public interface MutableConfigurationProvider {
/**
+ * Apply transactions which were not committed.
+ * @throws IOException if recovery fails
+ */
+ void recoverConf() throws IOException;
+
+ /**
* Update the scheduler configuration with the provided key value pairs.
* @param user User issuing the request
* @param confUpdate Key-value pairs for configurations to be updated.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index da395b7..6d2de7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -393,6 +393,9 @@ public class CapacityScheduler extends
@Override
public void serviceStart() throws Exception {
startSchedulerThreads();
+ if (this.csConfProvider instanceof MutableConfigurationProvider) {
+ ((MutableConfigurationProvider) csConfProvider).recoverConf();
+ }
super.serviceStart();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
new file mode 100644
index 0000000..1534685
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.fusesource.leveldbjni.JniDBFactory;
+import org.fusesource.leveldbjni.internal.NativeDB;
+import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBComparator;
+import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.DBIterator;
+import org.iq80.leveldb.Options;
+import org.iq80.leveldb.WriteBatch;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+
+/**
+ * A LevelDB implementation of {@link YarnConfigurationStore}.
+ */
+public class LeveldbConfigurationStore implements YarnConfigurationStore {
+
+ public static final Log LOG =
+ LogFactory.getLog(LeveldbConfigurationStore.class);
+
+ private static final String DB_NAME = "yarn-conf-store";
+ private static final String LOG_PREFIX = "log.";
+ private static final String LOG_COMMITTED_TXN = "committedTxn";
+
+ private DB db;
+ private long txnId = 0;
+ private long minTxn = 0;
+ private long maxLogs;
+ private Configuration conf;
+ private LinkedList<LogMutation> pendingMutations = new LinkedList<>();
+ private Timer compactionTimer;
+ private long compactionIntervalMsec;
+
+ @Override
+ public void initialize(Configuration config, Configuration schedConf)
+ throws IOException {
+ this.conf = config;
+ try {
+ this.db = initDatabase(schedConf);
+ this.txnId = Long.parseLong(new String(db.get(bytes(LOG_COMMITTED_TXN)),
+ StandardCharsets.UTF_8));
+ DBIterator itr = db.iterator();
+ itr.seek(bytes(LOG_PREFIX + txnId));
+ // Seek to first uncommitted log
+ itr.next();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ if (!new String(entry.getKey(), StandardCharsets.UTF_8)
+ .startsWith(LOG_PREFIX)) {
+ break;
+ }
+ pendingMutations.add(deserLogMutation(entry.getValue()));
+ }
+ // Get the earliest txnId stored in logs
+ itr.seekToFirst();
+ if (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ byte[] key = entry.getKey();
+ String logId = new String(key, StandardCharsets.UTF_8);
+ if (logId.startsWith(LOG_PREFIX)) {
+ minTxn = Long.parseLong(logId.substring(logId.indexOf('.') + 1));
+ }
+ }
+ this.maxLogs = config.getLong(
+ YarnConfiguration.RM_SCHEDCONF_LEVELDB_MAX_LOGS,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS);
+ this.compactionIntervalMsec = config.getLong(
+ YarnConfiguration.RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS,
+ YarnConfiguration
+ .DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS) * 1000;
+ startCompactionTimer();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ private DB initDatabase(Configuration config) throws Exception {
+ Path storeRoot = createStorageDir();
+ Options options = new Options();
+ options.createIfMissing(false);
+ options.comparator(new DBComparator() {
+ @Override
+ public int compare(byte[] key1, byte[] key2) {
+ String key1Str = new String(key1, StandardCharsets.UTF_8);
+ String key2Str = new String(key2, StandardCharsets.UTF_8);
+ int key1Txn = Integer.MAX_VALUE;
+ int key2Txn = Integer.MAX_VALUE;
+ if (key1Str.startsWith(LOG_PREFIX)) {
+ key1Txn = Integer.parseInt(key1Str.substring(
+ key1Str.indexOf('.') + 1));
+ }
+ if (key2Str.startsWith(LOG_PREFIX)) {
+ key2Txn = Integer.parseInt(key2Str.substring(
+ key2Str.indexOf('.') + 1));
+ }
+ // TODO txnId could overflow, in theory
+ if (key1Txn == Integer.MAX_VALUE && key2Txn == Integer.MAX_VALUE) {
+ if (key1Str.equals(key2Str) && key1Str.equals(LOG_COMMITTED_TXN)) {
+ return 0;
+ } else if (key1Str.equals(LOG_COMMITTED_TXN)) {
+ return -1;
+ } else if (key2Str.equals(LOG_COMMITTED_TXN)) {
+ return 1;
+ }
+ return key1Str.compareTo(key2Str);
+ }
+ return key1Txn - key2Txn;
+ }
+
+ @Override
+ public String name() {
+ return "logComparator";
+ }
+
+ public byte[] findShortestSeparator(byte[] start, byte[] limit) {
+ return start;
+ }
+
+ public byte[] findShortSuccessor(byte[] key) {
+ return key;
+ }
+ });
+ LOG.info("Using conf database at " + storeRoot);
+ File dbfile = new File(storeRoot.toString());
+ try {
+ db = JniDBFactory.factory.open(dbfile, options);
+ } catch (NativeDB.DBException e) {
+ if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
+ LOG.info("Creating conf database at " + dbfile);
+ options.createIfMissing(true);
+ try {
+ db = JniDBFactory.factory.open(dbfile, options);
+ // Write the initial scheduler configuration
+ WriteBatch initBatch = db.createWriteBatch();
+ for (Map.Entry<String, String> kv : config) {
+ initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
+ }
+ initBatch.put(bytes(LOG_COMMITTED_TXN), bytes("0"));
+ db.write(initBatch);
+ } catch (DBException dbErr) {
+ throw new IOException(dbErr.getMessage(), dbErr);
+ }
+ } else {
+ throw e;
+ }
+ }
+ return db;
+ }
+
+ private Path createStorageDir() throws IOException {
+ Path root = getStorageDir();
+ FileSystem fs = FileSystem.getLocal(conf);
+ fs.mkdirs(root, new FsPermission((short) 0700));
+ return root;
+ }
+
+ private Path getStorageDir() throws IOException {
+ String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
+ if (storePath == null) {
+ throw new IOException("No store location directory configured in " +
+ YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
+ }
+ return new Path(storePath, DB_NAME);
+ }
+
+ @Override
+ public synchronized long logMutation(LogMutation logMutation)
+ throws IOException {
+ logMutation.setId(++txnId);
+ WriteBatch logBatch = db.createWriteBatch();
+ logBatch.put(bytes(LOG_PREFIX + txnId), serLogMutation(logMutation));
+ if (txnId - minTxn >= maxLogs) {
+ logBatch.delete(bytes(LOG_PREFIX + minTxn));
+ minTxn++;
+ }
+ db.write(logBatch);
+ pendingMutations.add(logMutation);
+ return txnId;
+ }
+
+ @Override
+ public synchronized boolean confirmMutation(long id, boolean isValid)
+ throws IOException {
+ WriteBatch updateBatch = db.createWriteBatch();
+ if (isValid) {
+ LogMutation mutation = deserLogMutation(db.get(bytes(LOG_PREFIX + id)));
+ for (Map.Entry<String, String> changes :
+ mutation.getUpdates().entrySet()) {
+ if (changes.getValue() == null || changes.getValue().isEmpty()) {
+ updateBatch.delete(bytes(changes.getKey()));
+ } else {
+ updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue()));
+ }
+ }
+ }
+ updateBatch.put(bytes(LOG_COMMITTED_TXN), bytes(String.valueOf(id)));
+ db.write(updateBatch);
+ // Assumes logMutation and confirmMutation are done in the same
+ // synchronized method. For example,
+ // {@link MutableCSConfigurationProvider#mutateConfiguration(
+ // UserGroupInformation user, SchedConfUpdateInfo confUpdate)}
+ pendingMutations.removeFirst();
+ return true;
+ }
+
+ private byte[] serLogMutation(LogMutation mutation) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try (ObjectOutput oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(mutation);
+ oos.flush();
+ return baos.toByteArray();
+ }
+ }
+ private LogMutation deserLogMutation(byte[] mutation) throws IOException {
+ try (ObjectInput input = new ObjectInputStream(
+ new ByteArrayInputStream(mutation))) {
+ return (LogMutation) input.readObject();
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public synchronized Configuration retrieve() {
+ DBIterator itr = db.iterator();
+ itr.seek(bytes(LOG_COMMITTED_TXN));
+ Configuration config = new Configuration(false);
+ itr.next();
+ while (itr.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = itr.next();
+ config.set(new String(entry.getKey(), StandardCharsets.UTF_8),
+ new String(entry.getValue(), StandardCharsets.UTF_8));
+ }
+ return config;
+ }
+
+ @Override
+ public List<LogMutation> getPendingMutations() {
+ return pendingMutations;
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ return null; // unimplemented
+ }
+
+ // TODO below was taken from LeveldbRMStateStore, it can probably be
+ // refactored
+ private void startCompactionTimer() {
+ if (compactionIntervalMsec > 0) {
+ compactionTimer = new Timer(
+ this.getClass().getSimpleName() + " compaction timer", true);
+ compactionTimer.schedule(new CompactionTimerTask(),
+ compactionIntervalMsec, compactionIntervalMsec);
+ }
+ }
+
+ private class CompactionTimerTask extends TimerTask {
+ @Override
+ public void run() {
+ long start = Time.monotonicNow();
+ LOG.info("Starting full compaction cycle");
+ try {
+ db.compactRange(null, null);
+ } catch (DBException e) {
+ LOG.error("Error compacting database", e);
+ }
+ long duration = Time.monotonicNow() - start;
+ LOG.info("Full compaction cycle completed in " + duration + " msec");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 670c0f9..9ccc146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import com.google.common.base.Joiner;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,6 +51,9 @@ import java.util.Map;
public class MutableCSConfigurationProvider implements CSConfigurationProvider,
MutableConfigurationProvider {
+ public static final Log LOG =
+ LogFactory.getLog(MutableCSConfigurationProvider.class);
+
private Configuration schedConf;
private YarnConfigurationStore confStore;
private ConfigurationMutationACLPolicy aclMutationPolicy;
@@ -68,6 +73,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
this.confStore = new InMemoryConfigurationStore();
break;
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ this.confStore = new LeveldbConfigurationStore();
+ break;
default:
this.confStore = YarnConfigurationStoreFactory.getStore(config);
break;
@@ -82,6 +90,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ // After initializing confStore, the store may already have an existing
+ // configuration. Use this one.
+ schedConf = confStore.retrieve();
this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
.getPolicy(config);
aclMutationPolicy.init(config, rmContext);
@@ -97,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(UserGroupInformation user,
+ public synchronized void mutateConfiguration(UserGroupInformation user,
SchedConfUpdateInfo confUpdate) throws IOException {
if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
throw new AccessControlException("User is not admin of all modified" +
@@ -124,6 +135,31 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
confStore.confirmMutation(id, true);
}
+ @Override
+ public void recoverConf() throws IOException {
+ List<LogMutation> uncommittedLogs = confStore.getPendingMutations();
+ Configuration oldConf = new Configuration(schedConf);
+ for (LogMutation mutation : uncommittedLogs) {
+ for (Map.Entry<String, String> kv : mutation.getUpdates().entrySet()) {
+ if (kv.getValue() == null) {
+ schedConf.unset(kv.getKey());
+ } else {
+ schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ try {
+ rmContext.getScheduler().reinitialize(conf, rmContext);
+ } catch (IOException e) {
+ schedConf = oldConf;
+ confStore.confirmMutation(mutation.getId(), false);
+ LOG.info("Configuration mutation " + mutation.getId()
+ + " was rejected", e);
+ continue;
+ }
+ confStore.confirmMutation(mutation.getId(), true);
+ LOG.info("Configuration mutation " + mutation.getId()+ " was accepted");
+ }
+ }
private Map<String, String> constructKeyValueConfUpdate(
SchedConfUpdateInfo mutationInfo) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 22c0ef8..065c877 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import java.io.IOException;
+import java.io.Serializable;
import java.util.List;
import java.util.Map;
@@ -43,7 +45,7 @@ public interface YarnConfigurationStore {
* LogMutation encapsulates the fields needed for configuration mutation
* audit logging and recovery.
*/
- class LogMutation {
+ class LogMutation implements Serializable {
private Map<String, String> updates;
private String user;
private long id;
@@ -106,16 +108,19 @@ public interface YarnConfigurationStore {
* Initialize the configuration store.
* @param conf configuration to initialize store with
* @param schedConf Initial key-value configuration to persist
+ * @throws IOException if initialization fails
*/
- void initialize(Configuration conf, Configuration schedConf);
+ void initialize(Configuration conf, Configuration schedConf)
+ throws IOException;
/**
* Logs the configuration change to backing store. Generates an id associated
* with this mutation, sets it in {@code logMutation}, and returns it.
* @param logMutation configuration change to be persisted in write ahead log
* @return id which configuration store associates with this mutation
+ * @throws IOException if logging fails
*/
- long logMutation(LogMutation logMutation);
+ long logMutation(LogMutation logMutation) throws IOException;
/**
* Should be called after {@code logMutation}. Gets the pending mutation
@@ -130,8 +135,9 @@ public interface YarnConfigurationStore {
* @param isValid if true, update persisted configuration with mutation
* associated with {@code id}.
* @return true on success
+ * @throws IOException if mutation confirmation fails
*/
- boolean confirmMutation(long id, boolean isValid);
+ boolean confirmMutation(long id, boolean isValid) throws IOException;
/**
* Retrieve the persisted configuration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
index dff4e77..631ce65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestYarnConfigurationStore.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.Yar
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -43,7 +44,7 @@ public class TestYarnConfigurationStore {
}
@Test
- public void testInMemoryConfigurationStore() {
+ public void testInMemoryConfigurationStore() throws IOException {
confStore = new InMemoryConfigurationStore();
confStore.initialize(new Configuration(), schedConf);
assertEquals("val1", confStore.retrieve().get("key1"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-14920. KMSClientProvider
won't work with KMS delegation token retrieved from non-Java client.
Contributed by Xiaoyu Yao.
Posted by jh...@apache.org.
HADOOP-14920. KMSClientProvider won't work with KMS delegation token retrieved from non-Java client. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b08a1fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b08a1fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b08a1fc
Branch: refs/heads/YARN-5734
Commit: 2b08a1fc644904a37545107666efc25b3552542d
Parents: 72d22b7
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Oct 6 10:12:24 2017 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Fri Oct 6 10:12:24 2017 -0700
----------------------------------------------------------------------
.../DelegationTokenAuthenticationHandler.java | 5 +-
.../web/DelegationTokenAuthenticator.java | 1 +
.../delegation/web/DelegationTokenManager.java | 15 +++-
...tionTokenAuthenticationHandlerWithMocks.java | 72 ++++++++++++++------
4 files changed, 70 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b08a1fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index 106c6be..6ee59f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -272,8 +272,11 @@ public abstract class DelegationTokenAuthenticationHandler
}
String renewer = ServletUtils.getParameter(request,
KerberosDelegationTokenAuthenticator.RENEWER_PARAM);
+ String service = ServletUtils.getParameter(request,
+ KerberosDelegationTokenAuthenticator.SERVICE_PARAM);
try {
- Token<?> dToken = tokenManager.createToken(requestUgi, renewer);
+ Token<?> dToken = tokenManager.createToken(requestUgi, renewer,
+ service);
map = delegationTokenToJSON(dToken);
} catch (IOException ex) {
throw new AuthenticationException(ex.toString(), ex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b08a1fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
index 3dba1db..a1c6003 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticator.java
@@ -70,6 +70,7 @@ public abstract class DelegationTokenAuthenticator implements Authenticator {
public static final String DELEGATION_PARAM = "delegation";
public static final String TOKEN_PARAM = "token";
public static final String RENEWER_PARAM = "renewer";
+ public static final String SERVICE_PARAM = "service";
public static final String DELEGATION_TOKEN_JSON = "Token";
public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString";
public static final String RENEW_DELEGATION_TOKEN_JSON = "long";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b08a1fc/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
index b1a8d48..fd19b67 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenManager.java
@@ -160,7 +160,14 @@ public class DelegationTokenManager {
@SuppressWarnings("unchecked")
public Token<? extends AbstractDelegationTokenIdentifier> createToken(
UserGroupInformation ugi, String renewer) {
- LOG.debug("Creating token with ugi:{}, renewer:{}.", ugi, renewer);
+ return createToken(ugi, renewer, null);
+ }
+
+ @SuppressWarnings("unchecked")
+ public Token<? extends AbstractDelegationTokenIdentifier> createToken(
+ UserGroupInformation ugi, String renewer, String service) {
+ LOG.debug("Creating token with ugi:{}, renewer:{}, service:{}.",
+ ugi, renewer, service !=null ? service : "");
renewer = (renewer == null) ? ugi.getShortUserName() : renewer;
String user = ugi.getUserName();
Text owner = new Text(user);
@@ -173,7 +180,11 @@ public class DelegationTokenManager {
tokenIdentifier.setOwner(owner);
tokenIdentifier.setRenewer(new Text(renewer));
tokenIdentifier.setRealUser(realUser);
- return new Token(tokenIdentifier, secretManager);
+ Token token = new Token(tokenIdentifier, secretManager);
+ if (service != null) {
+ token.setService(new Text(service));
+ }
+ return token;
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b08a1fc/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
index f62a6ce..9357f48 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestDelegationTokenAuthenticationHandlerWithMocks.java
@@ -111,12 +111,21 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
@Test
public void testManagementOperations() throws Exception {
- testNonManagementOperation();
- testManagementOperationErrors();
- testGetToken(null, new Text("foo"));
- testGetToken("bar", new Text("foo"));
- testCancelToken();
- testRenewToken();
+ final Text testTokenKind = new Text("foo");
+ final String testRenewer = "bar";
+ final String testService = "192.168.64.101:8888";
+ testNonManagementOperation();
+ testManagementOperationErrors();
+ testGetToken(null, null, testTokenKind);
+ testGetToken(testRenewer, null, testTokenKind);
+ testCancelToken();
+ testRenewToken(testRenewer);
+
+ // Management operations against token requested with service parameter
+ Token<DelegationTokenIdentifier> testToken =
+ testGetToken(testRenewer, testService, testTokenKind);
+ testRenewToken(testToken, testRenewer);
+ testCancelToken(testToken);
}
private void testNonManagementOperation() throws Exception {
@@ -156,8 +165,8 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
Mockito.eq("mock"));
}
- private void testGetToken(String renewer, Text expectedTokenKind)
- throws Exception {
+ private Token<DelegationTokenIdentifier> testGetToken(String renewer,
+ String service, Text expectedTokenKind) throws Exception {
DelegationTokenAuthenticator.DelegationTokenOperation op =
DelegationTokenAuthenticator.DelegationTokenOperation.
GETDELEGATIONTOKEN;
@@ -173,10 +182,14 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
new StringWriter()));
Assert.assertFalse(handler.managementOperation(token, request, response));
- Mockito.when(request.getQueryString()).
- thenReturn(DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() +
- "&" + DelegationTokenAuthenticator.RENEWER_PARAM + "=" + renewer);
-
+ String queryString =
+ DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() + "&" +
+ DelegationTokenAuthenticator.RENEWER_PARAM + "=" + renewer;
+ if (service != null) {
+ queryString += "&" + DelegationTokenAuthenticator.SERVICE_PARAM + "="
+ + service;
+ }
+ Mockito.when(request.getQueryString()).thenReturn(queryString);
Mockito.reset(response);
Mockito.reset(token);
Mockito.when(token.getUserName()).thenReturn("user");
@@ -208,10 +221,25 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
dt.decodeFromUrlString(tokenStr);
handler.getTokenManager().verifyToken(dt);
Assert.assertEquals(expectedTokenKind, dt.getKind());
+ if (service != null) {
+ Assert.assertEquals(service, dt.getService().toString());
+ } else {
+ Assert.assertEquals(0, dt.getService().getLength());
+ }
+ return dt;
}
@SuppressWarnings("unchecked")
private void testCancelToken() throws Exception {
+ Token<DelegationTokenIdentifier> token =
+ (Token<DelegationTokenIdentifier>) handler.getTokenManager()
+ .createToken(UserGroupInformation.getCurrentUser(), "foo");
+ testCancelToken(token);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void testCancelToken(Token<DelegationTokenIdentifier> token)
+ throws Exception {
DelegationTokenAuthenticator.DelegationTokenOperation op =
DelegationTokenAuthenticator.DelegationTokenOperation.
CANCELDELEGATIONTOKEN;
@@ -228,9 +256,6 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
Mockito.contains("requires the parameter [token]"));
Mockito.reset(response);
- Token<DelegationTokenIdentifier> token =
- (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
- UserGroupInformation.getCurrentUser(), "foo");
Mockito.when(request.getQueryString()).thenReturn(
DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() + "&" +
DelegationTokenAuthenticator.TOKEN_PARAM + "=" +
@@ -249,7 +274,16 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
}
@SuppressWarnings("unchecked")
- private void testRenewToken() throws Exception {
+ private void testRenewToken(String testRenewer) throws Exception {
+ Token<DelegationTokenIdentifier> dToken = (Token<DelegationTokenIdentifier>)
+ handler.getTokenManager().createToken(
+ UserGroupInformation.getCurrentUser(), testRenewer);
+ testRenewToken(dToken, testRenewer);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void testRenewToken(Token<DelegationTokenIdentifier> dToken,
+ String testRenewer) throws Exception {
DelegationTokenAuthenticator.DelegationTokenOperation op =
DelegationTokenAuthenticator.DelegationTokenOperation.
RENEWDELEGATIONTOKEN;
@@ -270,7 +304,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
Mockito.reset(response);
AuthenticationToken token = Mockito.mock(AuthenticationToken.class);
- Mockito.when(token.getUserName()).thenReturn("user");
+ Mockito.when(token.getUserName()).thenReturn(testRenewer);
Assert.assertFalse(handler.managementOperation(token, request, response));
Mockito.verify(response).sendError(
Mockito.eq(HttpServletResponse.SC_BAD_REQUEST),
@@ -280,9 +314,7 @@ public class TestDelegationTokenAuthenticationHandlerWithMocks {
StringWriter writer = new StringWriter();
PrintWriter pwriter = new PrintWriter(writer);
Mockito.when(response.getWriter()).thenReturn(pwriter);
- Token<DelegationTokenIdentifier> dToken =
- (Token<DelegationTokenIdentifier>) handler.getTokenManager().createToken(
- UserGroupInformation.getCurrentUser(), "user");
+
Mockito.when(request.getQueryString()).
thenReturn(DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString() +
"&" + DelegationTokenAuthenticator.TOKEN_PARAM + "=" +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-7044.
TestContainerAllocation#testAMContainerAllocationWhenDNSUnavailable fails.
Posted by jh...@apache.org.
YARN-7044. TestContainerAllocation#testAMContainerAllocationWhenDNSUnavailable fails.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2df1b2ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2df1b2ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2df1b2ac
Branch: refs/heads/YARN-5734
Commit: 2df1b2ac0509ba10fff606ada7e9b3562c12dd16
Parents: 20e9ce3
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Oct 5 06:07:32 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Oct 5 06:07:32 2017 +0900
----------------------------------------------------------------------
.../scheduler/capacity/TestContainerAllocation.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df1b2ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 906febf..b1ca72a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.security.SecurityUtilTestHelper;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -294,11 +295,12 @@ public class TestContainerAllocation {
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExp,
- ContainerType containerType) {
+ ContainerType containerType, ExecutionType executionType) {
numRetries++;
return super.createContainerToken(containerId, containerVersion,
nodeId, appSubmitter, capability, priority, createTime,
- logAggregationContext, nodeLabelExp, containerType);
+ logAggregationContext, nodeLabelExp, containerType,
+ executionType);
}
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-6916. Moving logging APIs
over to slf4j in hadoop-yarn-server-common. Contributed by Bibin A Chundatt
and Akira Ajisaka.
Posted by jh...@apache.org.
YARN-6916. Moving logging APIs over to slf4j in hadoop-yarn-server-common. Contributed by Bibin A Chundatt and Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a877737
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a877737
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a877737
Branch: refs/heads/YARN-5734
Commit: 4a877737182808fe3392a116762ba59973a477a6
Parents: 107c177
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 4 06:06:36 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 4 06:06:39 2017 +0900
----------------------------------------------------------------------
.../hadoop/yarn/server/api/ServerRMProxy.java | 7 +++---
.../OpportunisticContainerAllocator.java | 26 +++++++++-----------
.../BaseContainerTokenSecretManager.java | 24 ++++++++----------
.../security/BaseNMTokenSecretManager.java | 26 ++++++++------------
.../server/sharedcache/SharedCacheUtil.java | 12 ++++-----
.../yarn/server/webapp/AppAttemptBlock.java | 7 +++---
.../hadoop/yarn/server/webapp/AppBlock.java | 6 ++---
.../hadoop/yarn/server/webapp/AppsBlock.java | 6 ++---
.../yarn/server/webapp/ContainerBlock.java | 7 +++---
.../server/webapp/ErrorsAndWarningsBlock.java | 14 +++--------
10 files changed, 60 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
index edec89f..cd92415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
@@ -21,17 +21,18 @@ package org.apache.hadoop.yarn.server.api;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.client.RMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ServerRMProxy<T> extends RMProxy<T> {
- private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServerRMProxy.class);
private ServerRMProxy() {
super();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
index 6fd5228..782dc02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/OpportunisticContainerAllocator.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.scheduler;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -42,6 +40,8 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -180,8 +180,8 @@ public class OpportunisticContainerAllocator {
}
}
- private static final Log LOG =
- LogFactory.getLog(OpportunisticContainerAllocator.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(OpportunisticContainerAllocator.class);
private static final ResourceCalculator RESOURCE_CALCULATOR =
new DominantResourceCalculator();
@@ -255,12 +255,11 @@ public class OpportunisticContainerAllocator {
appContext.getContainerIdGenerator(), appContext.getBlacklist(),
appAttId, appContext.getNodeMap(), userName, containers, anyAsk);
if (!containers.isEmpty()) {
- LOG.info("Opportunistic allocation requested for ["
- + "priority=" + anyAsk.getPriority()
- + ", allocationRequestId=" + anyAsk.getAllocationRequestId()
- + ", num_containers=" + anyAsk.getNumContainers()
- + ", capability=" + anyAsk.getCapability() + "]"
- + " allocated = " + containers.keySet());
+ LOG.info("Opportunistic allocation requested for [priority={}, "
+ + "allocationRequestId={}, num_containers={}, capability={}] "
+ + "allocated = {}", anyAsk.getPriority(),
+ anyAsk.getAllocationRequestId(), anyAsk.getNumContainers(),
+ anyAsk.getCapability(), containers.keySet());
}
}
return containers;
@@ -286,8 +285,7 @@ public class OpportunisticContainerAllocator {
}
if (nodesForScheduling.isEmpty()) {
LOG.warn("No nodes available for allocating opportunistic containers. [" +
- "allNodes=" + allNodes + ", " +
- "blacklist=" + blacklist + "]");
+ "allNodes={}, blacklist={}]", allNodes, blacklist);
return;
}
int numAllocated = 0;
@@ -305,9 +303,9 @@ public class OpportunisticContainerAllocator {
}
cList.add(container);
numAllocated++;
- LOG.info("Allocated [" + container.getId() + "] as opportunistic.");
+ LOG.info("Allocated [{}] as opportunistic.", container.getId());
}
- LOG.info("Allocated " + numAllocated + " opportunistic containers.");
+ LOG.info("Allocated {} opportunistic containers.", numAllocated);
}
private Container buildContainer(long rmIdentifier,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
index e73d07c..a5590fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
@@ -23,14 +23,14 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* SecretManager for ContainerTokens. Extended by both RM and NM and hence is
@@ -40,8 +40,8 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
public class BaseContainerTokenSecretManager extends
SecretManager<ContainerTokenIdentifier> {
- private static Log LOG = LogFactory
- .getLog(BaseContainerTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseContainerTokenSecretManager.class);
protected int serialNo = new SecureRandom().nextInt();
@@ -86,11 +86,9 @@ public class BaseContainerTokenSecretManager extends
@Override
public byte[] createPassword(ContainerTokenIdentifier identifier) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating password for " + identifier.getContainerID()
- + " for user " + identifier.getUser() + " to be run on NM "
- + identifier.getNmHostAddress());
- }
+ LOG.debug("Creating password for {} for user {} to be run on NM {}",
+ identifier.getContainerID(), identifier.getUser(),
+ identifier.getNmHostAddress());
this.readLock.lock();
try {
return createPassword(identifier.getBytes(),
@@ -114,11 +112,9 @@ public class BaseContainerTokenSecretManager extends
protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
MasterKeyData masterKey)
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving password for " + identifier.getContainerID()
- + " for user " + identifier.getUser() + " to be run on NM "
- + identifier.getNmHostAddress());
- }
+ LOG.debug("Retrieving password for {} for user {} to be run on NM {}",
+ identifier.getContainerID(), identifier.getUser(),
+ identifier.getNmHostAddress());
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
index 759b6f2..3cbd1dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
@@ -24,8 +24,6 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -35,12 +33,14 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BaseNMTokenSecretManager extends
SecretManager<NMTokenIdentifier> {
- private static Log LOG = LogFactory
- .getLog(BaseNMTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseNMTokenSecretManager.class);
protected int serialNo = new SecureRandom().nextInt();
@@ -71,12 +71,9 @@ public class BaseNMTokenSecretManager extends
@Override
protected byte[] createPassword(NMTokenIdentifier identifier) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("creating password for "
- + identifier.getApplicationAttemptId() + " for user "
- + identifier.getApplicationSubmitter() + " to run on NM "
- + identifier.getNodeId());
- }
+ LOG.debug("creating password for {} for user {} to run on NM {}",
+ identifier.getApplicationAttemptId(),
+ identifier.getApplicationSubmitter(), identifier.getNodeId());
readLock.lock();
try {
return createPassword(identifier.getBytes(),
@@ -99,12 +96,9 @@ public class BaseNMTokenSecretManager extends
protected byte[] retrivePasswordInternal(NMTokenIdentifier identifier,
MasterKeyData masterKey) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("creating password for "
- + identifier.getApplicationAttemptId() + " for user "
- + identifier.getApplicationSubmitter() + " to run on NM "
- + identifier.getNodeId());
- }
+ LOG.debug("retriving password for {} for user {} to run on NM {}",
+ identifier.getApplicationAttemptId(),
+ identifier.getApplicationSubmitter(), identifier.getNodeId());
return createPassword(identifier.getBytes(), masterKey.getSecretKey());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
index d3cf379..0e29432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/sharedcache/SharedCacheUtil.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.yarn.server.sharedcache;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A utility class that contains helper methods for dealing with the internal
@@ -34,7 +34,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Unstable
public class SharedCacheUtil {
- private static final Log LOG = LogFactory.getLog(SharedCacheUtil.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SharedCacheUtil.class);
@Private
public static int getCacheDepth(Configuration conf) {
@@ -44,9 +45,8 @@ public class SharedCacheUtil {
if (cacheDepth <= 0) {
LOG.warn("Specified cache depth was less than or equal to zero."
- + " Using default value instead. Default: "
- + YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL
- + ", Specified: " + cacheDepth);
+ + " Using default value instead. Default: {}, Specified: {}",
+ YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL, cacheDepth);
cacheDepth = YarnConfiguration.DEFAULT_SHARED_CACHE_NESTED_LEVEL;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index cad14b6..d663dc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -26,8 +26,6 @@ import java.util.Collection;
import java.util.List;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@@ -46,10 +44,13 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppAttemptBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppAttemptBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected ApplicationAttemptId appAttemptId = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index b429b5d..a8810a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -29,8 +29,6 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.UserGroupInformation;
@@ -67,10 +65,12 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppBlock.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AppBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
protected ApplicationId appID = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index e3a47de..b2dd4de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -34,8 +34,6 @@ import java.util.List;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.math.LongRange;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -51,10 +49,12 @@ import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AppsBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(AppsBlock.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AppsBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected EnumSet<YarnApplicationState> reqAppStates;
protected UserGroupInformation callerUGI;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
index b8c0bed..fd48dc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
@@ -23,8 +23,6 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -38,10 +36,13 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ContainerBlock extends HtmlBlock {
- private static final Log LOG = LogFactory.getLog(ContainerBlock.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerBlock.class);
protected ApplicationBaseProtocol appBaseProt;
@Inject
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a877737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
index 75f6fcd..4128546 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ErrorsAndWarningsBlock.java
@@ -19,11 +19,9 @@
package org.apache.hadoop.yarn.server.webapp;
import com.google.inject.Inject;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericsUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
@@ -60,8 +58,6 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
@Override
protected void render(Block html) {
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
-
boolean isAdmin = false;
UserGroupInformation callerUGI = this.getCallerUGI();
@@ -78,7 +74,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
return;
}
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
html.__(ErrorMetrics.class);
html.__(WarningMetrics.class);
html.div().button().$onclick("reloadPage()").b("View data for the last ")
@@ -180,8 +176,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
cutoffs.add((now - 43200 * 1000) / 1000);
cutoffs.add((now - 84600 * 1000) / 1000);
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
appender =
Log4jWarningErrorMetricsAppender.findAppender();
}
@@ -193,8 +188,7 @@ public class ErrorsAndWarningsBlock extends HtmlBlock {
@Override
protected void render(Block html) {
- Log log = LogFactory.getLog(ErrorsAndWarningsBlock.class);
- if (log instanceof Log4JLogger) {
+ if (GenericsUtil.isLog4jLogger(ErrorsAndWarningsBlock.class)) {
Hamlet.DIV<Hamlet> div =
html.div().$class("metrics").$style("padding-bottom: 20px");
div.h3(tableHeading).table("#metricsoverview").thead()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-5953:Create CLI for changing
YARN configurations. (Jonathan Hung via xgong)
Posted by jh...@apache.org.
YARN-5953:Create CLI for changing YARN configurations. (Jonathan Hung via xgong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/605ff23a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/605ff23a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/605ff23a
Branch: refs/heads/YARN-5734
Commit: 605ff23a3b8bb32ff16d47e38b7ba330eac6c239
Parents: d9a979a
Author: Xuan <xg...@apache.org>
Authored: Fri Jul 7 14:16:46 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 5 +
.../hadoop/yarn/client/cli/SchedConfCLI.java | 238 +++++++++++++++++++
.../yarn/client/cli/TestSchedConfCLI.java | 160 +++++++++++++
.../hadoop/yarn/webapp/dao/package-info.java | 27 +++
.../yarn/webapp/util/YarnWebServiceUtils.java | 14 ++
.../ConfigurationMutationACLPolicy.java | 2 +-
.../DefaultConfigurationMutationACLPolicy.java | 2 +-
.../scheduler/MutableConfScheduler.java | 2 +-
.../scheduler/MutableConfigurationProvider.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../conf/MutableCSConfigurationProvider.java | 4 +-
...ueueAdminConfigurationMutationACLPolicy.java | 4 +-
.../resourcemanager/webapp/RMWebServices.java | 1 +
.../webapp/dao/QueueConfigInfo.java | 4 +-
.../webapp/dao/SchedConfUpdateInfo.java | 18 +-
.../TestConfigurationMutationACLPolicies.java | 4 +-
.../TestMutableCSConfigurationProvider.java | 4 +-
.../TestRMWebServicesConfigurationMutation.java | 65 +++--
19 files changed, 507 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 52138c5..0063596 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -142,6 +142,9 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
;;
+ schedconf)
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
+ ;;
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 690badf..7ec9848 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -295,6 +295,11 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
+:schedconf
+ set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
@rem This changes %1, %2 etc. Hence those cannot be used after calling this.
:make_command_arguments
if "%1" == "--config" (
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
new file mode 100644
index 0000000..e17062e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -0,0 +1,238 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.client.cli;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * CLI for modifying scheduler configuration.
+ */
+@Public
+@Evolving
+public class SchedConfCLI extends Configured implements Tool {
+
+ private static final String ADD_QUEUES_OPTION = "addQueues";
+ private static final String REMOVE_QUEUES_OPTION = "removeQueues";
+ private static final String UPDATE_QUEUES_OPTION = "updateQueues";
+ private static final String GLOBAL_OPTIONS = "globalUpdates";
+ private static final String HELP_CMD = "help";
+
+ private static final String CONF_ERR_MSG = "Specify configuration key " +
+ "value as confKey=confVal.";
+
+ public SchedConfCLI() {
+ super(new YarnConfiguration());
+ }
+
+ public static void main(String[] args) throws Exception {
+ SchedConfCLI cli = new SchedConfCLI();
+ int exitCode = cli.run(args);
+ System.exit(exitCode);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options opts = new Options();
+
+ opts.addOption("add", ADD_QUEUES_OPTION, true,
+ "Add queues with configurations");
+ opts.addOption("remove", REMOVE_QUEUES_OPTION, true,
+ "Remove queues");
+ opts.addOption("update", UPDATE_QUEUES_OPTION, true,
+ "Update queue configurations");
+ opts.addOption("global", GLOBAL_OPTIONS, true,
+ "Update global scheduler configurations");
+ opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
+
+ int exitCode = -1;
+ CommandLine parsedCli = null;
+ try {
+ parsedCli = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ System.err.println("Missing argument for options");
+ printUsage();
+ return exitCode;
+ }
+
+ if (parsedCli.hasOption(HELP_CMD)) {
+ printUsage();
+ return 0;
+ }
+
+ boolean hasOption = false;
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ try {
+ if (parsedCli.hasOption(ADD_QUEUES_OPTION)) {
+ hasOption = true;
+ addQueues(parsedCli.getOptionValue(ADD_QUEUES_OPTION), updateInfo);
+ }
+ if (parsedCli.hasOption(REMOVE_QUEUES_OPTION)) {
+ hasOption = true;
+ removeQueues(parsedCli.getOptionValue(REMOVE_QUEUES_OPTION),
+ updateInfo);
+ }
+ if (parsedCli.hasOption(UPDATE_QUEUES_OPTION)) {
+ hasOption = true;
+ updateQueues(parsedCli.getOptionValue(UPDATE_QUEUES_OPTION),
+ updateInfo);
+ }
+ if (parsedCli.hasOption(GLOBAL_OPTIONS)) {
+ hasOption = true;
+ globalUpdates(parsedCli.getOptionValue(GLOBAL_OPTIONS), updateInfo);
+ }
+ } catch (IllegalArgumentException e) {
+ System.err.println(e.getMessage());
+ return -1;
+ }
+
+ if (!hasOption) {
+ System.err.println("Invalid Command Usage: ");
+ printUsage();
+ return -1;
+ }
+
+ Client webServiceClient = Client.create();
+ WebResource webResource = webServiceClient.resource(WebAppUtils.
+ getRMWebAppURLWithScheme(getConf()));
+ ClientResponse response = webResource.path("ws").path("v1").path("cluster")
+ .path("sched-conf").accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ if (response != null) {
+ if (response.getStatus() == Status.OK.getStatusCode()) {
+ System.out.println("Configuration changed successfully.");
+ return 0;
+ } else {
+ System.err.println("Configuration change unsuccessful: "
+ + response.getEntity(String.class));
+ }
+ } else {
+ System.err.println("Configuration change unsuccessful: null response");
+ }
+ return -1;
+ }
+
+ @VisibleForTesting
+ void addQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ ArrayList<QueueConfigInfo> queueConfigInfos = new ArrayList<>();
+ for (String arg : args.split(";")) {
+ queueConfigInfos.add(getQueueConfigInfo(arg));
+ }
+ updateInfo.setAddQueueInfo(queueConfigInfos);
+ }
+
+ @VisibleForTesting
+ void removeQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ List<String> queuesToRemove = Arrays.asList(args.split(","));
+ updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove));
+ }
+
+ @VisibleForTesting
+ void updateQueues(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ ArrayList<QueueConfigInfo> queueConfigInfos = new ArrayList<>();
+ for (String arg : args.split(";")) {
+ queueConfigInfos.add(getQueueConfigInfo(arg));
+ }
+ updateInfo.setUpdateQueueInfo(queueConfigInfos);
+ }
+
+ @VisibleForTesting
+ void globalUpdates(String args, SchedConfUpdateInfo updateInfo) {
+ if (args == null) {
+ return;
+ }
+ HashMap<String, String> globalUpdates = new HashMap<>();
+ for (String globalUpdate : args.split(",")) {
+ putKeyValuePair(globalUpdates, globalUpdate);
+ }
+ updateInfo.setGlobalParams(globalUpdates);
+ }
+
+ private QueueConfigInfo getQueueConfigInfo(String arg) {
+ String[] queueArgs = arg.split(",");
+ String queuePath = queueArgs[0];
+ Map<String, String> queueConfigs = new HashMap<>();
+ for (int i = 1; i < queueArgs.length; ++i) {
+ putKeyValuePair(queueConfigs, queueArgs[i]);
+ }
+ return new QueueConfigInfo(queuePath, queueConfigs);
+ }
+
+ private void putKeyValuePair(Map<String, String> kv, String args) {
+ String[] argParts = args.split("=");
+ if (argParts.length == 1) {
+ if (argParts[0].isEmpty() || !args.contains("=")) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ } else {
+ // key specified, but no value e.g. "confKey="
+ kv.put(argParts[0], null);
+ }
+ } else if (argParts.length > 2) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ } else {
+ if (argParts[0].isEmpty()) {
+ throw new IllegalArgumentException(CONF_ERR_MSG);
+ }
+ kv.put(argParts[0], argParts[1]);
+ }
+ }
+
+ private void printUsage() {
+ System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1,"
+ + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] "
+ + "[-remove queueRemovePath1,queueRemovePath2] "
+ + "[-update queueUpdatePath1,confKey1=confVal1] "
+ + "[-global globalConfKey1=globalConfVal1,"
+ + "globalConfKey2=globalConfVal2]");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
new file mode 100644
index 0000000..d2f0639
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -0,0 +1,160 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.client.cli;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Class for testing {@link SchedConfCLI}.
+ */
+public class TestSchedConfCLI {
+
+ private ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+
+ private ByteArrayOutputStream sysErrStream;
+ private PrintStream sysErr;
+
+ private SchedConfCLI cli;
+
+ @Before
+ public void setUp() {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = new PrintStream(sysOutStream);
+ System.setOut(sysOut);
+
+ sysErrStream = new ByteArrayOutputStream();
+ sysErr = new PrintStream(sysErrStream);
+ System.setErr(sysErr);
+
+ cli = new SchedConfCLI();
+ }
+
+ @Test(timeout = 10000)
+ public void testInvalidConf() throws Exception {
+ // conf pair with no key should be invalid
+ int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ exitCode = cli.run(new String[] {"-update", "root.a,=confVal"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+
+ exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"});
+ assertTrue("Should return an error code", exitCode != 0);
+ assertTrue(sysErrStream.toString().contains("Specify configuration key " +
+ "value as confKey=confVal."));
+ }
+
+ @Test(timeout = 10000)
+ public void testAddQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.addQueues("root.a,a1=aVal1,a2=aVal2," +
+ "a3=", schedUpdateInfo);
+ QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
+ assertEquals("root.a", addInfo.getQueue());
+ Map<String, String> params = addInfo.getParams();
+ assertEquals(3, params.size());
+ assertEquals("aVal1", params.get("a1"));
+ assertEquals("aVal2", params.get("a2"));
+ assertNull(params.get("a3"));
+
+ schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ assertEquals(2, schedUpdateInfo.getAddQueueInfo().size());
+ QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0);
+ assertEquals("root.b", bAddInfo.getQueue());
+ Map<String, String> bParams = bAddInfo.getParams();
+ assertEquals(1, bParams.size());
+ assertEquals("bVal1", bParams.get("b1"));
+ QueueConfigInfo cAddInfo = schedUpdateInfo.getAddQueueInfo().get(1);
+ assertEquals("root.c", cAddInfo.getQueue());
+ Map<String, String> cParams = cAddInfo.getParams();
+ assertEquals(1, cParams.size());
+ assertEquals("cVal1", cParams.get("c1"));
+ }
+
+ @Test(timeout = 10000)
+ public void testRemoveQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo);
+ List<String> removeInfo = schedUpdateInfo.getRemoveQueueInfo();
+ assertEquals(3, removeInfo.size());
+ assertEquals("root.a", removeInfo.get(0));
+ assertEquals("root.b", removeInfo.get(1));
+ assertEquals("root.c.c1", removeInfo.get(2));
+ }
+
+ @Test(timeout = 10000)
+ public void testUpdateQueues() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.updateQueues("root.a,a1=aVal1,a2=aVal2," +
+ "a3=", schedUpdateInfo);
+ QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
+ assertEquals("root.a", updateInfo.getQueue());
+ Map<String, String> params = updateInfo.getParams();
+ assertEquals(3, params.size());
+ assertEquals("aVal1", params.get("a1"));
+ assertEquals("aVal2", params.get("a2"));
+ assertNull(params.get("a3"));
+
+ schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size());
+ QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
+ assertEquals("root.b", bUpdateInfo.getQueue());
+ Map<String, String> bParams = bUpdateInfo.getParams();
+ assertEquals(1, bParams.size());
+ assertEquals("bVal1", bParams.get("b1"));
+ QueueConfigInfo cUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(1);
+ assertEquals("root.c", cUpdateInfo.getQueue());
+ Map<String, String> cParams = cUpdateInfo.getParams();
+ assertEquals(1, cParams.size());
+ assertEquals("cVal1", cParams.get("c1"));
+ }
+
+ @Test(timeout = 10000)
+ public void testGlobalUpdate() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2",
+ schedUpdateInfo);
+ Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams();
+ assertEquals(2, globalInfo.size());
+ assertEquals("schedVal1", globalInfo.get("schedKey1"));
+ assertEquals("schedVal2", globalInfo.get("schedKey2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
new file mode 100644
index 0000000..aec6762
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Data structures for scheduler configuration mutation info.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.webapp.dao;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
index 4167e21..1cf1e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/YarnWebServiceUtils.java
@@ -23,9 +23,14 @@ import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
import org.apache.hadoop.conf.Configuration;
import org.codehaus.jettison.json.JSONObject;
+import java.io.StringWriter;
+
/**
* This class contains several utility function which could be used to generate
* Restful calls to RM/NM/AHS.
@@ -59,4 +64,13 @@ public final class YarnWebServiceUtils {
.get(ClientResponse.class);
return response.getEntity(JSONObject.class);
}
+
+ @SuppressWarnings("rawtypes")
+ public static String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
index 3a388fe..5bc5874 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
/**
* Interface for determining whether configuration mutations are allowed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
index 6648668..1de6f6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
/**
* Default configuration mutation ACL policy. Checks if user is YARN admin.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 027d944..007dc29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 6b8306c..86be7c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 4d367e0..da395b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,11 +137,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index eb97260..670c0f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
index 0a82d50..ee53fd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.HashSet;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index b97e212..1da4e65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -199,6 +199,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Inject;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
index b20eda6..d1d91c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueConfigInfo.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
import java.util.HashMap;
import java.util.Map;
@@ -54,4 +54,4 @@ public class QueueConfigInfo {
return this.params;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
index b7c585e..bb84096 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedConfUpdateInfo.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
import java.util.ArrayList;
import java.util.HashMap;
@@ -54,16 +54,32 @@ public class SchedConfUpdateInfo {
return addQueueInfo;
}
+ public void setAddQueueInfo(ArrayList<QueueConfigInfo> addQueueInfo) {
+ this.addQueueInfo = addQueueInfo;
+ }
+
public ArrayList<String> getRemoveQueueInfo() {
return removeQueueInfo;
}
+ public void setRemoveQueueInfo(ArrayList<String> removeQueueInfo) {
+ this.removeQueueInfo = removeQueueInfo;
+ }
+
public ArrayList<QueueConfigInfo> getUpdateQueueInfo() {
return updateQueueInfo;
}
+ public void setUpdateQueueInfo(ArrayList<QueueConfigInfo> updateQueueInfo) {
+ this.updateQueueInfo = updateQueueInfo;
+ }
+
@XmlElementWrapper(name = "global-updates")
public HashMap<String, String> getGlobalParams() {
return global;
}
+
+ public void setGlobalParams(HashMap<String, String> globalInfo) {
+ this.global = globalInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
index 0f5a3d8..398e909 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 3216781..9104f16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/605ff23a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 5fbe36f..26ef1b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -22,8 +22,6 @@ import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.json.JSONJAXBContext;
-import com.sun.jersey.api.json.JSONMarshaller;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
@@ -35,11 +33,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -50,7 +49,6 @@ import javax.ws.rs.core.Response.Status;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.StringWriter;
import java.util.HashMap;
import java.util.Map;
@@ -183,8 +181,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -218,8 +216,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -244,8 +242,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -269,8 +267,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -300,8 +298,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -332,8 +330,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
@@ -360,8 +358,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -395,8 +393,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
@@ -413,8 +411,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
newCSConf = cs.getConfiguration();
@@ -443,8 +441,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -468,8 +466,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf =
@@ -483,8 +481,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("sched-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(updateInfo, SchedConfUpdateInfo.class),
- MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
newCSConf =
@@ -506,13 +504,4 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
super.tearDown();
}
-
- @SuppressWarnings("rawtypes")
- private String toJson(Object nsli, Class klass) throws Exception {
- StringWriter sw = new StringWriter();
- JSONJAXBContext ctx = new JSONJAXBContext(klass);
- JSONMarshaller jm = ctx.createJSONMarshaller();
- jm.marshallToJSON(nsli, sw);
- return sw.toString();
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-12455. WebHDFS - Adding
"snapshot enabled" status to ListStatus query result. Contributed by Ajay
Kumar.
Posted by jh...@apache.org.
HDFS-12455. WebHDFS - Adding "snapshot enabled" status to ListStatus query result. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/107c1777
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/107c1777
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/107c1777
Branch: refs/heads/YARN-5734
Commit: 107c177782a24a16c66113841f2fc5144f56207b
Parents: b913051
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Oct 3 13:02:12 2017 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Oct 3 13:02:12 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileStatus.java | 27 +++++++++++++++++++-
.../apache/hadoop/fs/protocolPB/PBHelper.java | 14 +++++++---
.../hadoop-common/src/main/proto/FSProtos.proto | 7 ++---
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 12 ++++++++-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 ++++
.../src/main/proto/hdfs.proto | 1 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 17 +++++++++---
...tNamenodeProtocolServerSideTranslatorPB.java | 1 +
.../server/namenode/FSDirStatAndListingOp.java | 7 +++++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 4 +++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 2 ++
.../hadoop/hdfs/TestDistributedFileSystem.java | 19 ++++++++++++++
12 files changed, 103 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index 8575439..d7c05d7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -60,7 +60,8 @@ public class FileStatus implements Writable, Comparable<Object>,
HAS_ACL,
HAS_CRYPT,
HAS_EC,
- };
+ SNAPSHOT_ENABLED
+ }
private static final Set<AttrFlags> NONE = Collections.<AttrFlags>emptySet();
private static Set<AttrFlags> flags(boolean acl, boolean crypt, boolean ec) {
if (!(acl || crypt || ec)) {
@@ -274,6 +275,15 @@ public class FileStatus implements Writable, Comparable<Object>,
}
/**
+ * Check if directory is Snapshot enabled or not.
+ *
+ * @return true if directory is snapshot enabled
+ */
+ public boolean isSnapshotEnabled() {
+ return attr.contains(AttrFlags.SNAPSHOT_ENABLED);
+ }
+
+ /**
* Get the owner of the file.
* @return owner of the file. The string could be empty if there is no
* notion of owner of a file in a filesystem or if it could not
@@ -331,6 +341,19 @@ public class FileStatus implements Writable, Comparable<Object>,
}
/**
+ * Sets Snapshot enabled flag.
+ *
+ * @param isSnapShotEnabled When true, SNAPSHOT_ENABLED flag is set
+ */
+ public void setSnapShotEnabledFlag(boolean isSnapShotEnabled) {
+ if (isSnapShotEnabled) {
+ attr.add(AttrFlags.SNAPSHOT_ENABLED);
+ } else {
+ attr.remove(AttrFlags.SNAPSHOT_ENABLED);
+ }
+ }
+
+ /**
* @return The contents of the symbolic link.
*/
public Path getSymlink() throws IOException {
@@ -486,4 +509,6 @@ public class FileStatus implements Writable, Comparable<Object>,
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
index aa7d8f5..23caf2e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/protocolPB/PBHelper.java
@@ -94,11 +94,15 @@ public final class PBHelper {
owner = proto.getOwner();
group = proto.getGroup();
int flags = proto.getFlags();
- return new FileStatus(length, isdir, blockReplication, blocksize,
- mtime, atime, permission, owner, group, symlink, path,
- (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0,
+ FileStatus fileStatus = new FileStatus(length, isdir, blockReplication,
+ blocksize, mtime, atime, permission, owner, group, symlink, path,
+ (flags & FileStatusProto.Flags.HAS_ACL_VALUE) != 0,
(flags & FileStatusProto.Flags.HAS_CRYPT_VALUE) != 0,
- (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0);
+ (flags & FileStatusProto.Flags.HAS_EC_VALUE) != 0);
+
+ fileStatus.setSnapShotEnabledFlag((flags & FileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE) != 0);
+ return fileStatus;
}
public static FileStatusProto convert(FileStatus stat) throws IOException {
@@ -124,6 +128,8 @@ public final class PBHelper {
flags |= stat.hasAcl() ? FileStatusProto.Flags.HAS_ACL_VALUE : 0;
flags |= stat.isEncrypted() ? FileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
flags |= stat.isErasureCoded() ? FileStatusProto.Flags.HAS_EC_VALUE : 0;
+ flags |= stat.isSnapshotEnabled() ? FileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE : 0;
bld.setFlags(flags);
return bld.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
index 6fe7980..5b8c45d 100644
--- a/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
+++ b/hadoop-common-project/hadoop-common/src/main/proto/FSProtos.proto
@@ -44,9 +44,10 @@ message FileStatusProto {
FT_SYMLINK = 3;
}
enum Flags {
- HAS_ACL = 0x01; // has ACLs
- HAS_CRYPT = 0x02; // encrypted
- HAS_EC = 0x04; // erasure coded
+ HAS_ACL = 0x01; // has ACLs
+ HAS_CRYPT = 0x02; // encrypted
+ HAS_EC = 0x04; // erasure coded
+ SNAPSHOT_ENABLED = 0x08; // snapshot enabled
}
required FileType fileType = 1;
required string path = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 8438b01..163881f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -56,7 +56,8 @@ public class HdfsFileStatus extends FileStatus {
public enum Flags {
HAS_ACL,
HAS_CRYPT,
- HAS_EC;
+ HAS_EC,
+ SNAPSHOT_ENABLED
}
private final EnumSet<Flags> flags;
@@ -244,6 +245,15 @@ public class HdfsFileStatus extends FileStatus {
return storagePolicy;
}
+ /**
+ * Check if directory is Snapshot enabled or not.
+ *
+ * @return true if directory is snapshot enabled
+ */
+ public boolean isSnapshotEnabled() {
+ return flags.contains(Flags.SNAPSHOT_ENABLED);
+ }
+
@Override
public boolean equals(Object o) {
// satisfy findbugs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index db97031..21b1e1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -1602,6 +1602,9 @@ public class PBHelperClient {
case HAS_EC:
f.add(HdfsFileStatus.Flags.HAS_EC);
break;
+ case SNAPSHOT_ENABLED:
+ f.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+ break;
default:
// ignore unknown
break;
@@ -2155,6 +2158,8 @@ public class PBHelperClient {
int flags = fs.hasAcl() ? HdfsFileStatusProto.Flags.HAS_ACL_VALUE : 0;
flags |= fs.isEncrypted() ? HdfsFileStatusProto.Flags.HAS_CRYPT_VALUE : 0;
flags |= fs.isErasureCoded() ? HdfsFileStatusProto.Flags.HAS_EC_VALUE : 0;
+ flags |= fs.isSnapshotEnabled() ? HdfsFileStatusProto.Flags
+ .SNAPSHOT_ENABLED_VALUE : 0;
builder.setFlags(flags);
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7769f68..c38a9be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -410,6 +410,7 @@ message HdfsFileStatusProto {
HAS_ACL = 0x01; // has ACLs
HAS_CRYPT = 0x02; // encrypted
HAS_EC = 0x04; // erasure coded
+ SNAPSHOT_ENABLED = 0x08; // SNAPSHOT ENABLED
}
required FileType fileType = 1;
required bytes path = 2; // local name of inode encoded java UTF8
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index b5880e9..3ff7a61 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -198,6 +198,7 @@ public class HttpFSFileSystem extends FileSystem
public static final String ENC_BIT_JSON = "encBit";
public static final String EC_BIT_JSON = "ecBit";
+ public static final String SNAPSHOT_BIT_JSON = "seBit";
public static final String DIRECTORY_LISTING_JSON = "DirectoryListing";
public static final String PARTIAL_LISTING_JSON = "partialListing";
@@ -1066,19 +1067,27 @@ public class HttpFSFileSystem extends FileSystem
final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON);
+ final Boolean snapshotEnabledBit = (Boolean) json.get(SNAPSHOT_BIT_JSON);
final boolean aBit = (aclBit != null) ? aclBit : false;
final boolean eBit = (encBit != null) ? encBit : false;
final boolean ecBit = (erasureBit != null) ? erasureBit : false;
- if (aBit || eBit || ecBit) {
+ final boolean seBit =
+ (snapshotEnabledBit != null) ? snapshotEnabledBit : false;
+ if (aBit || eBit || ecBit || seBit) {
// include this for compatibility with 2.x
FsPermissionExtension deprecatedPerm =
new FsPermissionExtension(permission, aBit, eBit, ecBit);
- return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ FileStatus fileStatus = new FileStatus(len, FILE_TYPE.DIRECTORY == type,
replication, blockSize, mTime, aTime, deprecatedPerm, owner, group,
null, path, aBit, eBit, ecBit);
+ if (seBit) {
+ fileStatus.setSnapShotEnabledFlag(seBit);
+ }
+ return fileStatus;
+ } else {
+ return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
+ replication, blockSize, mTime, aTime, permission, owner, group, path);
}
- return new FileStatus(len, FILE_TYPE.DIRECTORY == type,
- replication, blockSize, mTime, aTime, permission, owner, group, path);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 0527b3b..af0b154 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -948,6 +948,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+
public GetFileLinkInfoResponseProto getFileLinkInfo(RpcController controller,
GetFileLinkInfoRequestProto req) throws ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 3b3368d..7c0ba2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -413,6 +413,8 @@ class FSDirStatAndListingOp {
.unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
final boolean isErasureCoded = (ecPolicy != null);
+ boolean isSnapShottable = false;
+
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -433,6 +435,8 @@ class FSDirStatAndListingOp {
loc = new LocatedBlocks();
}
}
+ } else if (node.isDirectory()) {
+ isSnapShottable = node.asDirectory().isSnapshottable();
}
int childrenNum = node.isDirectory() ?
@@ -451,6 +455,9 @@ class FSDirStatAndListingOp {
if (isErasureCoded) {
flags.add(HdfsFileStatus.Flags.HAS_EC);
}
+ if(isSnapShottable){
+ flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
+ }
return createFileStatus(
size,
node.isDirectory(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 7fa818a..eae6adb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -135,6 +135,10 @@ public class JsonUtil {
if (status.isErasureCoded()) {
m.put("ecBit", true);
}
+ if (status.isSnapshotEnabled()) {
+ m.put("snapshotEnabled", status.isSnapshotEnabled());
+ }
+
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 84e8a57..73c6558 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -440,6 +440,7 @@ See also: [`newlength`](#New_Length), [FileSystem](../../api/org/apache/hadoop/f
"pathSuffix" : "",
"permission" : "777",
"replication" : 0,
+ "snapshotEnabled" : true
"type" : "DIRECTORY" //enum {FILE, DIRECTORY, SYMLINK}
}
}
@@ -485,6 +486,7 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
"pathSuffix" : "bar",
"permission" : "711",
"replication" : 0,
+ "snapshotEnabled" : true
"type" : "DIRECTORY"
},
...
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107c1777/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 987992e..6389f0d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1269,6 +1269,25 @@ public class TestDistributedFileSystem {
}
}
+ @Test
+ public void testListStatusOfSnapshotDirs() throws IOException {
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
+ .build();
+ try {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ dfs.create(new Path("/parent/test1/dfsclose/file-0"));
+ Path snapShotDir = new Path("/parent/test1/");
+ dfs.allowSnapshot(snapShotDir);
+
+ FileStatus status = dfs.getFileStatus(new Path("/parent/test1"));
+ assertTrue(status.isSnapshotEnabled());
+ status = dfs.getFileStatus(new Path("/parent/"));
+ assertFalse(status.isSnapshotEnabled());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
@Test(timeout=10000)
public void testDFSClientPeerReadTimeout() throws IOException {
final int timeout = 1000;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-7238. Documentation for API
based scheduler configuration management
Posted by jh...@apache.org.
YARN-7238. Documentation for API based scheduler configuration management
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e73e0e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e73e0e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e73e0e7
Branch: refs/heads/YARN-5734
Commit: 3e73e0e7eff7a033886b72177b9310369cbb8f72
Parents: 674a773
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Sep 25 17:47:32 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../src/site/markdown/CapacityScheduler.md | 24 ++-
.../src/site/markdown/ResourceManagerRest.md | 186 +++++++++++++++++++
.../src/site/markdown/YarnCommands.md | 13 ++
3 files changed, 221 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e73e0e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 6bb8489..d70f891 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -295,9 +295,29 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach
Changing Queue Configuration
----------------------------
-Changing queue properties and adding new queues is very simple. You need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*.
+Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API.
+
+### Changing queue configuration via file
+
+ To edit by file, you need to edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*.
$ vi $HADOOP_CONF_DIR/capacity-scheduler.xml
$ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues
-**Note:** Queues cannot be *deleted*, only addition of new queues is supported - the updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%.
+### Changing queue configuration via API
+
+ Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml.
+
+ | Property | Description |
+ |:---- |:---- |
+ | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. |
+ | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). |
+ | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
+ | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
+ | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
+ | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
+ | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
+
+ **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
+
+ See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e73e0e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 82f80f3..025e9a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -4430,3 +4430,189 @@ Response Body:
<remainingTimeInSeconds>90</remainingTimeInSeconds>
</timeout>
```
+
+Scheduler Configuration Mutation API
+--------------------------------
+
+The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy.
+
+
+### URI
+
+ * http://rm-http-address:port/ws/v1/cluster/sched-conf
+
+### HTTP Operations Supported
+
+ * PUT
+
+### Elements of the *schedConf* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| update-queue | object | A queue whose configurations should be updated |
+| add-queue | object | A queue to add to the scheduler along with this queue's configurations |
+| remove-queue | string | Full path name of a queue to remove |
+| global-updates | map | Map of key value pairs to update scheduler's global configuration |
+
+### PUT Request Examples
+
+Put requests are used to modify the scheduler configuration. A successful mutation results in a 200 response. A malformed request or one which resulted in an invalid scheduler configuration results in a 400 response.
+
+**Updating queue configuration(s)**
+
+Request for updating queue configurations.
+
+*Elements of the* update-queue *object*
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| queueName | string | Full path name of the queue to update |
+| params | map | A map of key value configuration pairs to update for this queue |
+
+Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10.
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <update-queue>
+ <queueName>root.default</queueName>
+ <params>
+ <entry>
+ <key>maximum-applications</key>
+ <value>100</value>
+ </entry>
+ <entry>
+ <key>minimum-user-limit-percent</key>
+ <value>10</value>
+ </entry>
+ </params>
+ </update-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+
+**Adding a queue**
+
+Request for adding queues/updating queue configurations.
+
+*Elements of the* add-queue *object*
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| queueName | string | Full path name of the queue to add |
+| params | map | A map of key value configuration pairs to set for this queue |
+
+Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.)
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <add-queue>
+ <queueName>root.a</queueName>
+ <params>
+ <entry>
+ <key>capacity</key>
+ <value>10</value>
+ </entry>
+ <entry>
+ <key>maximum-capacity</key>
+ <value>10</value>
+ </entry>
+ </params>
+ </add-queue>
+ <update-queue>
+ <queueName>root.default</queueName>
+ <params>
+ <entry>
+ <key>capacity</key>
+ <value>90</value>
+ </entry>
+ <entry>
+ <key>maximum-capacity</key>
+ <value>90</value>
+ </entry>
+ </params>
+ </update-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+**Removing queues**
+
+Request for removing queues from the queue hierarchy.
+
+Assuming we are using the capacity scheduler and the current queue configuration is three queues *root.default*, *root.a*, and *root.b*, this example removes both *root.a* and *root.b*. (More complex examples include removing a parent queue and its children.)
+
+**Note:** Queues must be put into `STOPPED` state before they are deleted. Any updated queue configuration should be a valid one i.e. queue-capacity at each *level* should be equal to 100%.
+
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <remove-queue>root.a</remove-queue>
+ <remove-queue>root.b</remove-queue>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
+
+**Updating global scheduler configurations**
+
+Request for updating global scheduler configurations. Assuming we are using the capacity scheduler, this example enables queue mappings. For global configuration updates, the full configuration key must be specified.
+
+HTTP Request:
+
+```xml
+ Accept: application/xml
+ PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ Content-Type: application/xml
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <schedConf>
+ <global-updates>
+ <entry>
+ <key>yarn.scheduler.capacity.queue-mappings-override.enable</key>
+ <value>true</value>
+ </entry>
+ </global-updates>
+ </schedConf>
+```
+
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Transfer-Encoding: chunked
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e73e0e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 5f430ec..2fc65bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -237,6 +237,19 @@ Usage:
Runs ResourceManager admin client
+### schedconf
+
+Usage: `yarn schedconf [options]`
+
+| COMMAND\_OPTIONS | Description |
+|:---- |:---- |
+| -add <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
+| -remove <queuePath1,queuePath2> | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
+| -update <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
+| -global <key1=val1,key2=val2> | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. |
+
+Updates scheduler configuration
+
### scmadmin
Usage: `yarn scmadmin [options] `
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HADOOP-14899. Restrict Access to
setPermission operation when authorization is enabled in WASB Contributed by
Kannapiran Srinivasan.
Posted by jh...@apache.org.
HADOOP-14899. Restrict Access to setPermission operation when authorization is enabled in WASB
Contributed by Kannapiran Srinivasan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/572cdb54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/572cdb54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/572cdb54
Branch: refs/heads/YARN-5734
Commit: 572cdb54632d8a29c4d9900f9e988ada8e3fe2ec
Parents: 49ae538
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 6 17:43:38 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 6 17:43:38 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 146 ++++++++--
.../hadoop-azure/src/site/markdown/index.md | 18 +-
.../TestNativeAzureFileSystemAuthorization.java | 276 +++++++++++++++++--
3 files changed, 391 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/572cdb54/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 2e5241d..07cfe72 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -79,6 +79,8 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
+
+import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -699,10 +701,29 @@ public class NativeAzureFileSystem extends FileSystem {
* Configuration property used to specify list of users that can perform
* chown operation when authorization is enabled in WASB.
*/
- public static final String AZURE_CHOWN_USERLIST_PROPERTY_NAME = "fs.azure.chown.allowed.userlist";
+ public static final String AZURE_CHOWN_USERLIST_PROPERTY_NAME =
+ "fs.azure.chown.allowed.userlist";
static final String AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
+ /**
+ * Configuration property used to specify list of daemon users that can
+ * perform chmod operation when authorization is enabled in WASB.
+ */
+ public static final String AZURE_DAEMON_USERLIST_PROPERTY_NAME =
+ "fs.azure.daemon.userlist";
+
+ static final String AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
+
+ /**
+ * Configuration property used to specify list of users that can perform
+ * chmod operation when authorization is enabled in WASB.
+ */
+ public static final String AZURE_CHMOD_USERLIST_PROPERTY_NAME =
+ "fs.azure.chmod.allowed.userlist";
+
+ static final String AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
+
static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME =
"fs.azure.block.location.impersonatedhost";
private static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT =
@@ -1189,7 +1210,9 @@ public class NativeAzureFileSystem extends FileSystem {
private DelegationTokenAuthenticatedURL authURL;
private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
private String credServiceUrl;
-
+ private List<String> chownAllowedUsers;
+ private List<String> chmodAllowedUsers;
+ private List<String> daemonUsers;
/**
* Configuration key to enable authorization support in WASB.
*/
@@ -1366,6 +1389,19 @@ public class NativeAzureFileSystem extends FileSystem {
this.authorizer =
new RemoteWasbAuthorizerImpl();
authorizer.init(conf);
+
+ this.chmodAllowedUsers =
+ Arrays.asList(conf.getTrimmedStrings(
+ AZURE_CHMOD_USERLIST_PROPERTY_NAME,
+ AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE));
+ this.chownAllowedUsers =
+ Arrays.asList(conf.getTrimmedStrings(
+ AZURE_CHOWN_USERLIST_PROPERTY_NAME,
+ AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE));
+ this.daemonUsers =
+ Arrays.asList(conf.getTrimmedStrings(
+ AZURE_DAEMON_USERLIST_PROPERTY_NAME,
+ AZURE_DAEMON_USERLIST_PROPERTY_DEFAULT_VALUE));
}
if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) {
@@ -3446,6 +3482,27 @@ public class NativeAzureFileSystem extends FileSystem {
if (metadata == null) {
throw new FileNotFoundException("File doesn't exist: " + p);
}
+
+ // If authorization is enabled, check if the user is
+ // part of chmod allowed list or a daemon user or owner of the file/folder
+ if (azureAuthorization) {
+ UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
+
+ // Check if the user is part of chown allowed list or a daemon user.
+ if (!isAllowedUser(currentUgi.getShortUserName(), chmodAllowedUsers)
+ && !isAllowedUser(currentUgi.getShortUserName(), daemonUsers)) {
+
+ //Check if the user is the owner of the file.
+ String owner = metadata.getPermissionStatus().getUserName();
+ if (!currentUgi.getShortUserName().equals(owner)) {
+ throw new WasbAuthorizationException(
+ String.format("user '%s' does not have the privilege to "
+ + "change the permission of files/folders.",
+ currentUgi.getShortUserName()));
+ }
+ }
+ }
+
permission = applyUMask(permission,
metadata.isDir() ? UMaskApplyMode.ChangeExistingDirectory
: UMaskApplyMode.ChangeExistingFile);
@@ -3491,26 +3548,14 @@ public class NativeAzureFileSystem extends FileSystem {
* to change the ownership of file/folder
*/
if (this.azureAuthorization && username != null) {
- String[] listOfUsers = getConf().getTrimmedStrings(AZURE_CHOWN_USERLIST_PROPERTY_NAME,
- AZURE_CHOWN_USERLIST_PROPERTY_DEFAULT_VALUE);
- boolean shouldSkipUserCheck = listOfUsers.length == 1 && listOfUsers[0].equals("*");
- // skip the check if the chown allowed users config value is set as '*'
- if (!shouldSkipUserCheck) {
- UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- UserGroupInformation actualUgi = currentUgi.getRealUser();
+ UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- if (actualUgi == null) {
- actualUgi = currentUgi;
- }
-
- List<String> userList = Arrays.asList(listOfUsers);
- if (userList.contains("*")) {
- throw new IllegalArgumentException("User list must contain "
- + "either '*' or list of user names, but not both.");
- } else if (!userList.contains(actualUgi.getShortUserName())) {
- throw new WasbAuthorizationException(String.format("user '%s' does not have the privilege to change the ownership of files/folders.",
- actualUgi.getShortUserName()));
- }
+ if (!isAllowedUser(currentUgi.getShortUserName(),
+ chownAllowedUsers)) {
+ throw new WasbAuthorizationException(
+ String.format("user '%s' does not have the privilege to change "
+ + "the ownership of files/folders.",
+ currentUgi.getShortUserName()));
}
}
@@ -3528,6 +3573,38 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
+ /**
+ * Is the user allowed?
+ * <ol>
+ * <li>No user: false</li>
+ * <li>Empty list: false</li>
+ * <li>List == ["*"]: true</li>
+ * <li>Otherwise: is the user in the list?</li>
+ * </ol>
+ * @param username user to check; may be null
+ * @param userList list of users; may be null or empty
+ * @return
+ * @throws IllegalArgumentException if the userList is invalid.
+ */
+ private boolean isAllowedUser(String username, List<String> userList) {
+
+ if (null == userList || userList.isEmpty()) {
+ return false;
+ }
+
+ boolean shouldSkipUserCheck = userList.size() == 1
+ && userList.get(0).equals("*");
+
+ // skip the check if the allowed users config value is set as '*'
+ if (!shouldSkipUserCheck) {
+ Preconditions.checkArgument(!userList.contains("*"),
+ "User list must contain either '*' or a list of user names,"
+ + " but not both.");
+ return userList.contains(username);
+ }
+ return true;
+ }
+
@Override
public synchronized void close() throws IOException {
if (isClosed) {
@@ -3779,4 +3856,31 @@ public class NativeAzureFileSystem extends FileSystem {
}
return owner;
}
+
+ /**
+ * Helper method to update the chownAllowedUsers in tests.
+ * @param chownAllowedUsers list of chown allowed users
+ */
+ @VisibleForTesting
+ void updateChownAllowedUsers(List<String> chownAllowedUsers) {
+ this.chownAllowedUsers = chownAllowedUsers;
+ }
+
+ /**
+ * Helper method to update the chmodAllowedUsers in tests.
+ * @param chmodAllowedUsers list of chmod allowed users
+ */
+ @VisibleForTesting
+ void updateChmodAllowedUsers(List<String> chmodAllowedUsers) {
+ this.chmodAllowedUsers = chmodAllowedUsers;
+ }
+
+ /**
+ * Helper method to update the daemonUsers in tests.
+ * @param daemonUsers list of daemon users
+ */
+ @VisibleForTesting
+ void updateDaemonUsers(List<String> daemonUsers) {
+ this.daemonUsers = daemonUsers;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/572cdb54/hadoop-tools/hadoop-azure/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
index 876d7cc..3f29a6e 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md
@@ -463,7 +463,7 @@ The service is expected to return a response in JSON format for GETDELEGATIONTOK
When authorization is enabled, only the users listed in the following configuration
are allowed to change the owning user of files/folders in WASB. The configuration
-value takes a comma seperated list of user names who are allowed to perform chown.
+value takes a comma separated list of user names who are allowed to perform chown.
```xml
<property>
@@ -471,6 +471,22 @@ value takes a comma seperated list of user names who are allowed to perform chow
<value>user1,user2</value>
</property>
```
+### chmod behaviour when authorization is enabled in WASB
+
+When authorization is enabled, only the owner and the users listed in the
+following configurations are allowed to change the permissions of files/folders in WASB.
+The configuration value takes a comma separated list of user names who are allowed to perform chmod.
+
+```xml
+<property>
+ <name>fs.azure.daemon.userlist</name>
+ <value>user1,user2</value>
+</property>
+<property>
+ <name>fs.azure.chmod.allowed.userlist</name>
+ <value>userA,userB</value>
+</property>
+```
Caching of both SAS keys and Authorization responses can be enabled using the following setting:
The cache settings are applicable only when fs.azure.authorization is enabled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/572cdb54/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
index 4e7b6fb..9a5e6e0 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java
@@ -21,10 +21,14 @@ package org.apache.hadoop.fs.azure;
import java.io.FileNotFoundException;
import java.security.PrivilegedExceptionAction;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -49,6 +53,8 @@ import static org.junit.Assert.assertEquals;
public class TestNativeAzureFileSystemAuthorization
extends AbstractWasbTestBase {
+ private static final short FULL_PERMISSION_WITH_STICKYBIT = 1777;
+
@VisibleForTesting
protected MockWasbAuthorizerImpl authorizer;
@@ -66,6 +72,8 @@ public class TestNativeAzureFileSystemAuthorization
conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URLS, "http://localhost/");
conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "user1 , user2");
conf.set(KEY_AUTH_SERVICE_CACHING_ENABLE, "false");
+ conf.set(NativeAzureFileSystem.AZURE_CHMOD_USERLIST_PROPERTY_NAME, "user1 , user2");
+ conf.set(NativeAzureFileSystem.AZURE_DAEMON_USERLIST_PROPERTY_NAME, "hive , hcs , yarn");
return conf;
}
@@ -925,7 +933,7 @@ public class TestNativeAzureFileSystemAuthorization
// create child with owner as dummyUser
UserGroupInformation dummyUser = UserGroupInformation.createUserForTesting(
- "dummyUser", new String[] {"dummygroup"});
+ "user1", new String[] {"dummygroup"});
dummyUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
@@ -1226,16 +1234,13 @@ public class TestNativeAzureFileSystemAuthorization
*/
@Test
public void testSetOwnerThrowsForUnauthorisedUsers() throws Throwable {
-
- expectedEx.expect(WasbAuthorizationException.class);
-
Path testPath = new Path("/testSetOwnerNegative");
authorizer.addAuthRuleForOwner("/", WRITE, true);
- authorizer.addAuthRuleForOwner(testPath.toString(), READ, true);
+ authorizer.addAuthRuleForOwner("/", READ, true);
fs.updateWasbAuthorizer(authorizer);
- String owner = null;
+ final String owner;
UserGroupInformation unauthorisedUser = UserGroupInformation.createUserForTesting(
"unauthoriseduser", new String[] {"group1"});
try {
@@ -1246,13 +1251,17 @@ public class TestNativeAzureFileSystemAuthorization
unauthorisedUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
+ try {
fs.setOwner(testPath, "newowner", null);
- return null;
+ fail("Failing test because setOwner call was expected to throw");
+ } catch (WasbAuthorizationException wex) {
+ // check that the owner is not modified
+ assertOwnerEquals(testPath, owner);
+ }
+ return null;
}
});
} finally {
- // check that the owner is not modified
- assertEquals(owner, fs.getFileStatus(testPath).getOwner());
fs.delete(testPath, false);
}
}
@@ -1288,8 +1297,8 @@ public class TestNativeAzureFileSystemAuthorization
@Override
public Void run() throws Exception {
fs.setOwner(testPath, newOwner, newGroup);
- assertEquals(newOwner, fs.getFileStatus(testPath).getOwner());
- assertEquals(newGroup, fs.getFileStatus(testPath).getGroup());
+ assertOwnerEquals(testPath, newOwner);
+ assertEquals(newGroup, fs.getFileStatus(testPath).getGroup());
return null;
}
});
@@ -1305,19 +1314,17 @@ public class TestNativeAzureFileSystemAuthorization
* */
@Test
public void testSetOwnerSucceedsForAnyUserWhenWildCardIsSpecified() throws Throwable {
+ fs.updateChownAllowedUsers(Collections.singletonList("*"));
+ final Path testPath = new Path("/testSetOwnerPositiveWildcard");
Configuration conf = fs.getConf();
- conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "*");
- fs.setConf(conf);
- Path testPath = new Path("/testSetOwnerPositiveWildcard");
-
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
- String newOwner = "newowner";
- String newGroup = "newgroup";
+ final String newOwner = "newowner";
+ final String newGroup = "newgroup";
UserGroupInformation user = UserGroupInformation.createUserForTesting(
"anyuser", new String[]{"group1"});
@@ -1334,7 +1341,7 @@ public class TestNativeAzureFileSystemAuthorization
@Override
public Void run() throws Exception {
fs.setOwner(testPath, newOwner, newGroup);
- assertEquals(newOwner, fs.getFileStatus(testPath).getOwner());
+ assertOwnerEquals(testPath, newOwner);
assertEquals(newGroup, fs.getFileStatus(testPath).getGroup());
return null;
}
@@ -1350,20 +1357,16 @@ public class TestNativeAzureFileSystemAuthorization
*/
@Test
public void testSetOwnerFailsForIllegalSetup() throws Throwable {
+ fs.updateChownAllowedUsers(Arrays.asList("user1", "*"));
- expectedEx.expect(IllegalArgumentException.class);
+ final Path testPath = new Path("/testSetOwnerFailsForIllegalSetup");
Configuration conf = fs.getConf();
- conf.set(NativeAzureFileSystem.AZURE_CHOWN_USERLIST_PROPERTY_NAME, "user1, *");
- fs.setConf(conf);
- Path testPath = new Path("/testSetOwnerFailsForIllegalSetup");
-
authorizer.init(conf);
authorizer.addAuthRuleForOwner("/", WRITE, true);
authorizer.addAuthRuleForOwner(testPath.getParent().toString(), READ, true);
fs.updateWasbAuthorizer(authorizer);
- String owner = null;
UserGroupInformation user = UserGroupInformation.createUserForTesting(
"anyuser", new String[]{"group1"});
try {
@@ -1371,18 +1374,22 @@ public class TestNativeAzureFileSystemAuthorization
fs.mkdirs(testPath);
ContractTestUtils.assertPathExists(fs, "test path does not exist", testPath);
- owner = fs.getFileStatus(testPath).getOwner();
+ final String owner = fs.getFileStatus(testPath).getOwner();
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
+ try {
fs.setOwner(testPath, "newowner", null);
+ fail("Failing test because setOwner call was expected to throw");
+ } catch (IllegalArgumentException iex) {
+ // check that the owner is not modified
+ assertOwnerEquals(testPath, owner);
+ }
return null;
}
});
} finally {
- // check that the owner is not modified
- assertEquals(owner, fs.getFileStatus(testPath).getOwner());
fs.delete(testPath, false);
}
}
@@ -1433,4 +1440,219 @@ public class TestNativeAzureFileSystemAuthorization
fs.delete(testPath, true);
}
}
+
+ /**
+ * Negative test for setPermission when Authorization is enabled.
+ */
+ @Test
+ public void testSetPermissionThrowsForUnauthorisedUsers() throws Throwable {
+ //setPermission is called by a user who is not a daemon user
+ //and not chmodAllowedUsers and not owner of the file/folder.
+ //This test validates a authorization exception during setPermission call
+ testSetPermission("/testSetPermissionNegative", null, null, "unauthorizeduser",
+ true, false);
+ }
+
+ /**
+ * Positive test for setPermission when Authorization is enabled.
+ */
+ @Test
+ public void testSetPermissionForAuthorisedUsers() throws Throwable {
+ //user1 is already part of chmodAllowedUsers.
+ //This test validates the allowed user can do setPermission
+ testSetPermission("/testSetPermissionPositive", null, null, "user1",
+ false, false);
+ }
+
+ /**
+ * Positive test for setPermission as owner when Authorization is enabled.
+ */
+ @Test
+ public void testSetPermissionForOwner() throws Throwable {
+ //setPermission is called by the owner and expect a success
+ //during setPermission call
+ testSetPermission("/testSetPermissionPositiveOwner",
+ null, null, null, false, false);
+ }
+
+ /**
+ * Test setPermission when wildcard is specified in allowed user list.
+ */
+ @Test
+ public void testSetPermissionWhenWildCardInAllowedUserList() throws Throwable {
+ //Allow all to setPermission and expect a success
+ //during setPermission call
+ List<String> chmodAllowedUsers = Collections.singletonList("*");
+
+ testSetPermission("/testSetPermissionWhenWildCardInAllowedUserList",
+ chmodAllowedUsers, null, "testuser", false, false);
+ }
+
+ /**
+ * Test setPermission when invalid configuration value for allowed user list
+ * i.e. wildcard character and a username.
+ */
+ @Test
+ public void testSetPermissionForInvalidAllowedUserList() throws Throwable {
+ //Setting up an invalid chmodAllowedUsers and expects a failure
+ //during setPermission call
+ List<String> chmodAllowedUsers = Arrays.asList("*", "testuser");
+
+ testSetPermission("/testSetPermissionForInvalidAllowedUserList",
+ chmodAllowedUsers, null, "testuser", true, true);
+ }
+
+ /**
+ * Test setPermission for a daemon user.
+ */
+ @Test
+ public void testSetPermissionForDaemonUser() throws Throwable {
+ //hive user is already setup as daemon user.
+ //This test validates the daemon user can do setPermission
+ testSetPermission("/testSetPermissionForDaemonUser", null,
+ null, "hive", false, false);
+ }
+
+ /**
+ * Test setPermission when invalid configuration value for daemon user list
+ * i.e. wildcard character and a daemon username.
+ */
+ @Test
+ public void testSetPermissionForInvalidDaemonUserList() throws Throwable {
+
+ List<String> daemonUsers = Arrays.asList("*", "hive");
+
+ testSetPermission("/testSetPermissionForInvalidDaemonUserList", null,
+ daemonUsers, "testuser", true, true);
+
+ }
+
+ /**
+ * Helper method to test setPermission scenarios. This method handles both positive
+ * and negative scenarios of setPermission tests
+ */
+ private void testSetPermission(String path,
+ List<String> chmodAllowedUsers,
+ List<String> daemonUsers,
+ String user,
+ boolean isSetPermissionFailureCase,
+ boolean isInvalidSetup) throws Throwable {
+
+ final FsPermission filePermission;
+
+ final Path testPath = new Path(path);
+ final FsPermission newPermission = new FsPermission(FULL_PERMISSION_WITH_STICKYBIT);
+ authorizer.addAuthRule("/", WRITE, getCurrentUserShortName(), true);
+ authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
+ fs.updateWasbAuthorizer(authorizer);
+
+ if (chmodAllowedUsers != null && !chmodAllowedUsers.isEmpty()) {
+ fs.updateChmodAllowedUsers(chmodAllowedUsers);
+ }
+
+ if (daemonUsers != null && !daemonUsers.isEmpty()) {
+ fs.updateDaemonUsers(daemonUsers);
+ }
+
+ UserGroupInformation testUser = (user != null) ? UserGroupInformation.createUserForTesting(
+ user, new String[] {"testgrp"}) : null;
+ try {
+ fs.mkdirs(testPath);
+ ContractTestUtils.assertPathExists(fs, "test path does not exist",
+ testPath);
+ filePermission = fs.getFileStatus(testPath).getPermission();
+
+ if (isSetPermissionFailureCase) {
+ executeSetPermissionFailure(testUser, testPath, filePermission,
+ newPermission, isInvalidSetup);
+ } else {
+ executeSetPermissionSuccess(testUser, testPath, filePermission,
+ newPermission);
+ }
+
+ } finally {
+ fs.delete(testPath, false);
+ }
+ }
+
+ /**
+ * This method expects a failure while invoking setPermission call
+ * and validates whether the failure is as expected
+ *
+ */
+ private void executeSetPermissionFailure(UserGroupInformation testUser,
+ Path testPath, FsPermission oldPermission, FsPermission newPermission,
+ boolean isInvalidSetup)
+ throws Throwable {
+ testUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ try {
+ //READ access required for getFileStatus
+ authorizer.addAuthRule("/", READ, getCurrentUserShortName(), true);
+ fs.setPermission(testPath, newPermission);
+ fail("Failing test because setPermission was expected to throw");
+
+ } catch (IllegalArgumentException iex) {
+ if (!isInvalidSetup) {
+ //fail if IllegalArgumentException is not expected
+ fail("Failing test because IllegalArgumentException"
+ + " is not expected to throw");
+ }
+ // check that the file permission is not modified.
+ assertPermissionEquals(testPath, oldPermission);
+ } catch (WasbAuthorizationException wex) {
+ if (isInvalidSetup) {
+ //fail if WasbAuthorizationException is not expected
+ fail("Failing test because WasbAuthorizationException"
+ + " is not expected to throw");
+ }
+ // check that the file permission is not modified.
+ assertPermissionEquals(testPath, oldPermission);
+ }
+ return null;
+ }
+ });
+ }
+
+ /**
+ * This method expects a success while invoking setPermission call
+ * and validates whether the new permissions are set
+ *
+ */
+ private void executeSetPermissionSuccess(UserGroupInformation testUser,
+ Path testPath, FsPermission oldPermission, FsPermission newPermission)
+ throws Throwable {
+ //If user is given, then use doAs
+ if (testUser != null) {
+ testUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ fs.setPermission(testPath, newPermission);
+ return null;
+ }
+ });
+ } else {
+ //If user is not given, then run in current user context
+ fs.setPermission(testPath, newPermission);
+ }
+
+ // check that the file permission is modified
+ assertPermissionEquals(testPath, newPermission);
+ // check old permission is not equals to new permission
+ assertNotEquals(newPermission, oldPermission);
+ }
+
+ private void assertPermissionEquals(Path path, FsPermission newPermission)
+ throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals("Wrong permissions in " + status,
+ newPermission, status.getPermission());
+ }
+
+ private void assertOwnerEquals(Path path, String owner) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals("Wrong owner in " + status, owner, status.getOwner());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-6840. Implement zookeeper
based store for scheduler configuration updates. (Jonathan Hung via wangda)
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79fb0a58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
new file mode 100644
index 0000000..3cfa8da
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -0,0 +1,312 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Tests {@link ZKConfigurationStore}.
+ */
+public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
+
+ public static final Log LOG =
+ LogFactory.getLog(TestZKConfigurationStore.class);
+
+ private static final int ZK_TIMEOUT_MS = 10000;
+ private TestingServer curatorTestingServer;
+ private CuratorFramework curatorFramework;
+ private ResourceManager rm;
+
+ public static TestingServer setupCuratorServer() throws Exception {
+ TestingServer curatorTestingServer = new TestingServer();
+ curatorTestingServer.start();
+ return curatorTestingServer;
+ }
+
+ public static CuratorFramework setupCuratorFramework(
+ TestingServer curatorTestingServer) throws Exception {
+ CuratorFramework curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(curatorTestingServer.getConnectString())
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+ return curatorFramework;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ curatorTestingServer = setupCuratorServer();
+ curatorFramework = setupCuratorFramework(curatorTestingServer);
+
+ conf.set(CommonConfigurationKeys.ZK_ADDRESS,
+ curatorTestingServer.getConnectString());
+ rm = new MockRM(conf);
+ rm.start();
+ rmContext = rm.getRMContext();
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ rm.stop();
+ curatorFramework.close();
+ curatorTestingServer.stop();
+ }
+
+ @Test
+ public void testVersioning() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ assertEquals(ZKConfigurationStore.CURRENT_VERSION_INFO,
+ confStore.getConfStoreVersion());
+ }
+
+ @Test
+ public void testPersistConfiguration() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ // Create a new configuration store, and check for old configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ }
+
+
+ @Test
+ public void testPersistUpdatedConfiguration() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.retrieve().get("key"));
+
+ Map<String, String> update = new HashMap<>();
+ update.put("key", "val");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update, TEST_USER);
+ confStore.logMutation(mutation);
+ confStore.confirmMutation(true);
+ assertEquals("val", confStore.retrieve().get("key"));
+
+ // Create a new configuration store, and check for updated configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ }
+
+ @Test
+ public void testMaxLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
+ confStore.initialize(conf, schedConf, rmContext);
+ LinkedList<YarnConfigurationStore.LogMutation> logs =
+ ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(0, logs.size());
+
+ Map<String, String> update1 = new HashMap<>();
+ update1.put("key1", "val1");
+ YarnConfigurationStore.LogMutation mutation =
+ new YarnConfigurationStore.LogMutation(update1, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ confStore.confirmMutation(true);
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+
+ Map<String, String> update2 = new HashMap<>();
+ update2.put("key2", "val2");
+ mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+
+ // Next update should purge first update from logs.
+ Map<String, String> update3 = new HashMap<>();
+ update3.put("key3", "val3");
+ mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
+ confStore.logMutation(mutation);
+ logs = ((ZKConfigurationStore) confStore).getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ }
+
+ public Configuration createRMHAConf(String rmIds, String rmId,
+ int adminPort) {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.ZK_CONFIGURATION_STORE);
+ conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS,
+ curatorTestingServer.getConnectString());
+ conf.set(YarnConfiguration.RM_HA_ID, rmId);
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ for (String rpcAddress :
+ YarnConfiguration.getServiceAddressConfKeys(conf)) {
+ for (String id : HAUtil.getRMHAIds(conf)) {
+ conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
+ }
+ }
+ conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId),
+ "localhost:" + adminPort);
+ return conf;
+ }
+
+ /**
+ * When failing over, new active RM should read from current state of store,
+ * including any updates when the new active RM was in standby.
+ * @throws Exception
+ */
+ @Test
+ public void testFailoverReadsFromUpdatedStore() throws Exception {
+ HAServiceProtocol.StateChangeRequestInfo req =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+
+ Configuration conf1 = createRMHAConf("rm1,rm2", "rm1", 1234);
+ ResourceManager rm1 = new MockRM(conf1);
+ rm1.start();
+ rm1.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm1.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertNull(((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+
+ Configuration conf2 = createRMHAConf("rm1,rm2", "rm2", 5678);
+ ResourceManager rm2 = new MockRM(conf2);
+ rm2.start();
+ assertEquals("RM should be Standby",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ // Update configuration on RM1
+ SchedConfUpdateInfo schedConfUpdateInfo = new SchedConfUpdateInfo();
+ schedConfUpdateInfo.getGlobalParams().put("key", "val");
+ MutableConfigurationProvider confProvider = ((MutableConfScheduler)
+ rm1.getResourceScheduler()).getMutableConfProvider();
+ UserGroupInformation user = UserGroupInformation
+ .createUserForTesting(TEST_USER, new String[0]);
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+ rm1.getResourceScheduler().reinitialize(conf1, rm1.getRMContext());
+ assertEquals("val", ((MutableConfScheduler) rm1.getResourceScheduler())
+ .getConfiguration().get("key"));
+ confProvider.confirmPendingMutation(true);
+ assertEquals("val", ((MutableCSConfigurationProvider) confProvider)
+ .getConfStore().retrieve().get("key"));
+ // Next update is not persisted, it should not be recovered
+ schedConfUpdateInfo.getGlobalParams().put("key", "badVal");
+ confProvider.logAndApplyMutation(user, schedConfUpdateInfo);
+
+ // Start RM2 and verifies it starts with updated configuration
+ rm2.getRMContext().getRMAdminService().transitionToActive(req);
+ assertEquals("RM with ZKStore didn't start",
+ Service.STATE.STARTED, rm2.getServiceState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
+ if (HAServiceProtocol.HAServiceState.ACTIVE ==
+ rm1.getRMContext().getRMAdminService().getServiceStatus()
+ .getState()) {
+ Thread.sleep(100);
+ }
+ }
+ assertEquals("RM should have been fenced",
+ HAServiceProtocol.HAServiceState.STANDBY,
+ rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
+ assertEquals("RM should be Active",
+ HAServiceProtocol.HAServiceState.ACTIVE,
+ rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
+
+ assertEquals("val", ((MutableCSConfigurationProvider) (
+ (CapacityScheduler) rm2.getResourceScheduler())
+ .getMutableConfProvider()).getConfStore().retrieve().get("key"));
+ assertEquals("val", ((MutableConfScheduler) rm2.getResourceScheduler())
+ .getConfiguration().get("key"));
+ // Transition to standby will set RM's HA status and then reinitialize in
+ // a separate thread. Despite asserting for STANDBY state, it's
+ // possible for reinitialization to be unfinished. Wait here for it to
+ // finish, otherwise closing rm1 will close zkManager and the unfinished
+ // reinitialization will throw an exception.
+ Thread.sleep(10000);
+ rm1.close();
+ rm2.close();
+ }
+
+ @Override
+ public YarnConfigurationStore createConfStore() {
+ return new ZKConfigurationStore();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-7285. ContainerExecutor
always launches with priorities due to yarn-default property. Contributed by
Jason Lowe.
Posted by jh...@apache.org.
YARN-7285. ContainerExecutor always launches with priorities due to yarn-default property. Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0046bb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0046bb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0046bb5
Branch: refs/heads/YARN-5734
Commit: d0046bb5c6919c9620ffa5330eb37b39484151ad
Parents: 7e76f85
Author: Naganarasimha <na...@apache.org>
Authored: Fri Oct 6 00:06:25 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Oct 6 00:07:53 2017 +0530
----------------------------------------------------------------------
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 4 ++--
.../server/nodemanager/TestLinuxContainerExecutorWithMocks.java | 3 ---
2 files changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0046bb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8453dc7..8b22fa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1869,10 +1869,10 @@
<property>
<description>
Adjustment to the container OS scheduling priority. In Linux, passed
- directly to the nice command.
+ directly to the nice command. If unspecified then containers are launched
+ without any explicit OS priority.
</description>
<name>yarn.nodemanager.container-executor.os.sched.priority.adjustment</name>
- <value>0</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0046bb5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 9fa22d8..ad675b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -247,9 +247,6 @@ public class TestLinuxContainerExecutorWithMocks {
// make sure the command doesn't contain the nice -n since priority
// not specified
List<String> command = new ArrayList<String>();
- Configuration conf = mockExec.getConf();
- conf.unset(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
- mockExec.setConf(conf);
mockExec.addSchedPriorityCommand(command);
assertEquals("addSchedPriority should be empty", 0, command.size());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-7279. Fix typo in helper
message of ContainerLauncher. Contributed by Elek, Marton.
Posted by jh...@apache.org.
YARN-7279. Fix typo in helper message of ContainerLauncher. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/592bf2d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/592bf2d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/592bf2d5
Branch: refs/heads/YARN-5734
Commit: 592bf2d550a07ea5c5df3ba0ab2952c34d941b4b
Parents: e6e614e
Author: Sunil G <su...@apache.org>
Authored: Thu Oct 5 10:52:00 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Oct 5 10:52:00 2017 +0530
----------------------------------------------------------------------
.../nodemanager/containermanager/launcher/ContainerLaunch.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/592bf2d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index f63b1b8..f67dce7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -642,7 +642,7 @@ public class ContainerLaunch implements Callable<Integer> {
.append(" <value>HADOOP_MAPRED_HOME=${full path of your hadoop "
+ "distribution directory}</value>\n")
.append("</property>\n<property>\n")
- .append(" <name>mapreduce.reduce.e nv</name>\n")
+ .append(" <name>mapreduce.reduce.env</name>\n")
.append(" <value>HADOOP_MAPRED_HOME=${full path of your hadoop "
+ "distribution directory}</value>\n")
.append("</property>\n");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-7211. AMSimulator in SLS
does't work due to responseId refactoring.(Contributed by Botong Huang via
Yufei Gu)
Posted by jh...@apache.org.
YARN-7211. AMSimulator in SLS does't work due to responseId refactoring.(Contributed by Botong Huang via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79e37dc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79e37dc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79e37dc6
Branch: refs/heads/YARN-5734
Commit: 79e37dc6dba44e7d1d913593304db7a8516056aa
Parents: 67e1216
Author: Yufei Gu <yu...@apache.org>
Authored: Tue Oct 3 15:08:29 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Tue Oct 3 15:10:50 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e37dc6/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
index c69805e..72698ea 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java
@@ -83,7 +83,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
RecordFactoryProvider.getRecordFactory(null);
// response queue
protected final BlockingQueue<AllocateResponse> responseQueue;
- protected int RESPONSE_ID = 1;
+ private int responseId = 0;
// user name
protected String user;
// queue name
@@ -274,7 +274,7 @@ public abstract class AMSimulator extends TaskRunner.Task {
List<ContainerId> toRelease) {
AllocateRequest allocateRequest =
recordFactory.newRecordInstance(AllocateRequest.class);
- allocateRequest.setResponseId(RESPONSE_ID ++);
+ allocateRequest.setResponseId(responseId++);
allocateRequest.setAskList(ask);
allocateRequest.setReleaseList(toRelease);
return allocateRequest;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-7258. Add Node and Rack Hints
to Opportunistic Scheduler. (Kartheek Muthyala via asuresh).
Posted by jh...@apache.org.
YARN-7258. Add Node and Rack Hints to Opportunistic Scheduler. (Kartheek Muthyala via asuresh).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b733348d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b733348d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b733348d
Branch: refs/heads/YARN-5734
Commit: b733348dde18a242e6c9074c512116a8baf1d281
Parents: 9288206
Author: Arun Suresh <as...@apache.org>
Authored: Mon Oct 2 18:01:51 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Oct 5 09:58:04 2017 -0700
----------------------------------------------------------------------
.../yarn/api/records/ResourceRequest.java | 16 +
.../api/impl/TestDistributedScheduling.java | 649 ---------------
.../TestOpportunisticContainerAllocation.java | 784 -------------------
...TestOpportunisticContainerAllocationE2E.java | 784 +++++++++++++++++++
.../server/api/protocolrecords/RemoteNode.java | 35 +
.../impl/pb/RemoteNodePBImpl.java | 19 +
.../OpportunisticContainerAllocator.java | 333 ++++++--
.../OpportunisticContainerContext.java | 68 +-
.../yarn_server_common_service_protos.proto | 1 +
.../TestOpportunisticContainerAllocator.java | 599 ++++++++++++++
...pportunisticContainerAllocatorAMService.java | 8 +-
...pportunisticContainerAllocatorAMService.java | 2 +
12 files changed, 1781 insertions(+), 1517 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 21fa15f..beb3380 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -240,6 +240,22 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
}
/**
+ * Set the <code>executionTypeRequest</code> of the request with 'ensure
+ * execution type' flag set to true.
+ * @see ResourceRequest#setExecutionTypeRequest(
+ * ExecutionTypeRequest)
+ * @param executionType <code>executionType</code> of the request.
+ * @return {@link ResourceRequestBuilder}
+ */
+ @Public
+ @Evolving
+ public ResourceRequestBuilder executionType(ExecutionType executionType) {
+ resourceRequest.setExecutionTypeRequest(
+ ExecutionTypeRequest.newInstance(executionType, true));
+ return this;
+ }
+
+ /**
* Set the <code>allocationRequestId</code> of the request.
* @see ResourceRequest#setAllocationRequestId(long)
* @param allocationRequestId
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
deleted file mode 100644
index 00f5e03..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestDistributedScheduling.java
+++ /dev/null
@@ -1,649 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.client.api.impl;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NMToken;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Validates End2End Distributed Scheduling flow which includes the AM
- * specifying OPPORTUNISTIC containers in its resource requests,
- * the AMRMProxyService on the NM, the DistributedScheduler RequestInterceptor
- * on the NM and the DistributedSchedulingProtocol used by the framework to talk
- * to the OpportunisticContainerAllocatorAMService running on the RM.
- */
-public class TestDistributedScheduling extends BaseAMRMProxyE2ETest {
-
- private static final Log LOG =
- LogFactory.getLog(TestDistributedScheduling.class);
-
- protected MiniYARNCluster cluster;
- protected YarnClient rmClient;
- protected ApplicationMasterProtocol client;
- protected Configuration conf;
- protected Configuration yarnConf;
- protected ApplicationAttemptId attemptId;
- protected ApplicationId appId;
-
- @Before
- public void doBefore() throws Exception {
- cluster = new MiniYARNCluster("testDistributedSchedulingE2E", 1, 1, 1);
-
- conf = new YarnConfiguration();
- conf.setBoolean(YarnConfiguration.
- OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
- conf.setBoolean(YarnConfiguration.DIST_SCHEDULING_ENABLED, true);
- conf.setInt(YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH,
- 10);
- cluster.init(conf);
- cluster.start();
- yarnConf = cluster.getConfig();
-
- // the client has to connect to AMRMProxy
- yarnConf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_AMRM_PROXY_ADDRESS);
- rmClient = YarnClient.createYarnClient();
- rmClient.init(yarnConf);
- rmClient.start();
-
- // Submit application
- attemptId = createApp(rmClient, cluster, conf);
- appId = attemptId.getApplicationId();
- client = createAMRMProtocol(rmClient, appId, cluster, yarnConf);
- }
-
- @After
- public void doAfter() throws Exception {
- if (client != null) {
- try {
- client.finishApplicationMaster(FinishApplicationMasterRequest
- .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
- rmClient.killApplication(attemptId.getApplicationId());
- attemptId = null;
- } catch (Exception e) {
- }
- }
- if (rmClient != null) {
- try {
- rmClient.stop();
- } catch (Exception e) {
- }
- }
- if (cluster != null) {
- try {
- cluster.stop();
- } catch (Exception e) {
- }
- }
- }
-
-
- /**
- * Validates if Allocate Requests containing only OPPORTUNISTIC container
- * requests are satisfied instantly.
- *
- * @throws Exception
- */
- @Test(timeout = 60000)
- public void testOpportunisticExecutionTypeRequestE2E() throws Exception {
- LOG.info("testDistributedSchedulingE2E - Register");
-
- RegisterApplicationMasterResponse responseRegister =
- client.registerApplicationMaster(RegisterApplicationMasterRequest
- .newInstance(NetUtils.getHostname(), 1024, ""));
-
- Assert.assertNotNull(responseRegister);
- Assert.assertNotNull(responseRegister.getQueue());
- Assert.assertNotNull(responseRegister.getApplicationACLs());
- Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
- Assert
- .assertNotNull(responseRegister.getContainersFromPreviousAttempts());
- Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
- Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
-
- // Wait until the RM has been updated and verify
- Map<ApplicationId, RMApp> rmApps =
- cluster.getResourceManager().getRMContext().getRMApps();
- boolean rmUpdated = false;
- for (int i=0; i<10 && !rmUpdated; i++) {
- sleep(100);
- RMApp rmApp = rmApps.get(appId);
- if (rmApp.getState() == RMAppState.RUNNING) {
- rmUpdated = true;
- }
- }
- RMApp rmApp = rmApps.get(appId);
- Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
-
- LOG.info("testDistributedSchedulingE2E - Allocate");
-
- AllocateRequest request =
- createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
-
- // Replace 'ANY' requests with OPPORTUNISTIC aks and remove
- // everything else
- List<ResourceRequest> newAskList = new ArrayList<>();
- for (ResourceRequest rr : request.getAskList()) {
- if (ResourceRequest.ANY.equals(rr.getResourceName())) {
- ResourceRequest newRR = ResourceRequest.newInstance(rr
- .getPriority(), rr.getResourceName(),
- rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(),
- rr.getNodeLabelExpression(),
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true));
- newAskList.add(newRR);
- }
- }
- request.setAskList(newAskList);
-
- AllocateResponse allocResponse = client.allocate(request);
- Assert.assertNotNull(allocResponse);
-
- // Ensure that all the requests are satisfied immediately
- Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
-
- // Verify that the allocated containers are OPPORTUNISTIC
- for (Container allocatedContainer : allocResponse
- .getAllocatedContainers()) {
- ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
- .newContainerTokenIdentifier(
- allocatedContainer.getContainerToken());
- Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
- containerTokenIdentifier.getExecutionType());
- }
-
- // Check that the RM sees OPPORTUNISTIC containers
- ResourceScheduler scheduler = cluster.getResourceManager()
- .getResourceScheduler();
- for (Container allocatedContainer : allocResponse
- .getAllocatedContainers()) {
- ContainerId containerId = allocatedContainer.getId();
- RMContainer rmContainer = scheduler.getRMContainer(containerId);
- Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
- rmContainer.getExecutionType());
- }
-
- LOG.info("testDistributedSchedulingE2E - Finish");
- }
-
- /**
- * Validates if Allocate Requests containing both GUARANTEED and OPPORTUNISTIC
- * container requests works as expected.
- *
- * @throws Exception
- */
- @Test(timeout = 60000)
- public void testMixedExecutionTypeRequestE2E() throws Exception {
- LOG.info("testDistributedSchedulingE2E - Register");
-
- RegisterApplicationMasterResponse responseRegister =
- client.registerApplicationMaster(RegisterApplicationMasterRequest
- .newInstance(NetUtils.getHostname(), 1024, ""));
-
- Assert.assertNotNull(responseRegister);
- Assert.assertNotNull(responseRegister.getQueue());
- Assert.assertNotNull(responseRegister.getApplicationACLs());
- Assert.assertNotNull(responseRegister.getClientToAMTokenMasterKey());
- Assert
- .assertNotNull(responseRegister.getContainersFromPreviousAttempts());
- Assert.assertNotNull(responseRegister.getSchedulerResourceTypes());
- Assert.assertNotNull(responseRegister.getMaximumResourceCapability());
-
- RMApp rmApp =
- cluster.getResourceManager().getRMContext().getRMApps().get(appId);
- Assert.assertEquals(RMAppState.RUNNING, rmApp.getState());
-
- LOG.info("testDistributedSchedulingE2E - Allocate");
-
- AllocateRequest request =
- createAllocateRequest(rmClient.getNodeReports(NodeState.RUNNING));
- List<ResourceRequest> askList = request.getAskList();
- List<ResourceRequest> newAskList = new ArrayList<>(askList);
-
- // Duplicate all ANY requests marking them as opportunistic
- for (ResourceRequest rr : askList) {
- if (ResourceRequest.ANY.equals(rr.getResourceName())) {
- ResourceRequest newRR = ResourceRequest.newInstance(rr
- .getPriority(), rr.getResourceName(),
- rr.getCapability(), rr.getNumContainers(), rr.getRelaxLocality(),
- rr.getNodeLabelExpression(),
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true));
- newAskList.add(newRR);
- }
- }
- request.setAskList(newAskList);
-
- AllocateResponse allocResponse = client.allocate(request);
- Assert.assertNotNull(allocResponse);
-
- // Ensure that all the requests are satisfied immediately
- Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
-
- // Verify that the allocated containers are OPPORTUNISTIC
- for (Container allocatedContainer : allocResponse
- .getAllocatedContainers()) {
- ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
- .newContainerTokenIdentifier(
- allocatedContainer.getContainerToken());
- Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
- containerTokenIdentifier.getExecutionType());
- }
-
- request.setAskList(new ArrayList<ResourceRequest>());
- request.setResponseId(request.getResponseId() + 1);
-
- Thread.sleep(1000);
-
- // RM should allocate GUARANTEED containers within 2 calls to allocate()
- allocResponse = client.allocate(request);
- Assert.assertNotNull(allocResponse);
- Assert.assertEquals(2, allocResponse.getAllocatedContainers().size());
-
- // Verify that the allocated containers are GUARANTEED
- for (Container allocatedContainer : allocResponse
- .getAllocatedContainers()) {
- ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils
- .newContainerTokenIdentifier(
- allocatedContainer.getContainerToken());
- Assert.assertEquals(ExecutionType.GUARANTEED,
- containerTokenIdentifier.getExecutionType());
- }
-
- LOG.info("testDistributedSchedulingE2E - Finish");
- }
-
- /**
- * Validates if AMRMClient can be used with Distributed Scheduling turned on.
- *
- * @throws Exception
- */
- @Test(timeout = 120000)
- @SuppressWarnings("unchecked")
- public void testAMRMClient() throws Exception {
- AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
- try {
- Priority priority = Priority.newInstance(1);
- Priority priority2 = Priority.newInstance(2);
- Resource capability = Resource.newInstance(1024, 1);
-
- List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
- String node = nodeReports.get(0).getNodeId().getHost();
- String rack = nodeReports.get(0).getRackName();
- String[] nodes = new String[]{node};
- String[] racks = new String[]{rack};
-
- // start am rm client
- amClient = new AMRMClientImpl(client);
- amClient.init(yarnConf);
- amClient.start();
- amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
-
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2,
- 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2,
- 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2,
- 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- RemoteRequestsTable<ContainerRequest> remoteRequestsTable =
- amClient.getTable(0);
- ProfileCapability profileCapability =
- ProfileCapability.newInstance(capability);
-
- int containersRequestedNode = remoteRequestsTable.get(priority,
- node, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- int containersRequestedRack = remoteRequestsTable.get(priority,
- rack, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- int containersRequestedAny = remoteRequestsTable.get(priority,
- ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
- .remoteRequest.getNumContainers();
- int oppContainersRequestedAny =
- remoteRequestsTable.get(priority2, ResourceRequest.ANY,
- ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
- .getNumContainers();
-
- assertEquals(2, containersRequestedNode);
- assertEquals(2, containersRequestedRack);
- assertEquals(2, containersRequestedAny);
- assertEquals(1, oppContainersRequestedAny);
-
- assertEquals(4, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- // RM should allocate container within 2 calls to allocate()
- int allocatedContainerCount = 0;
- int iterationsLeft = 10;
- Set<ContainerId> releases = new TreeSet<>();
-
- amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
- amClient.getNMTokenCache().numberOfTokensInCache());
- HashMap<String, Token> receivedNMTokens = new HashMap<>();
-
- while (allocatedContainerCount <
- (containersRequestedAny + oppContainersRequestedAny)
- && iterationsLeft-- > 0) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- allocatedContainerCount += allocResponse.getAllocatedContainers()
- .size();
- for (Container container : allocResponse.getAllocatedContainers()) {
- ContainerId rejectContainerId = container.getId();
- releases.add(rejectContainerId);
- }
-
- for (NMToken token : allocResponse.getNMTokens()) {
- String nodeID = token.getNodeId().toString();
- receivedNMTokens.put(nodeID, token.getToken());
- }
-
- if (allocatedContainerCount < containersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
- }
- }
-
- assertEquals(allocatedContainerCount,
- containersRequestedAny + oppContainersRequestedAny);
- for (ContainerId rejectContainerId : releases) {
- amClient.releaseAssignedContainer(rejectContainerId);
- }
- assertEquals(3, amClient.release.size());
- assertEquals(0, amClient.ask.size());
-
- // need to tell the AMRMClient that we dont need these resources anymore
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority2,
- 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- assertEquals(4, amClient.ask.size());
-
- // test RPC exception handling
- amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability,
- nodes, racks, priority));
- amClient.addContainerRequest(new AMRMClient.ContainerRequest(capability,
- nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority2,
- 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- final AMRMClient amc = amClient;
- ApplicationMasterProtocol realRM = amClient.rmClient;
- try {
- ApplicationMasterProtocol mockRM = mock(ApplicationMasterProtocol
- .class);
- when(mockRM.allocate(any(AllocateRequest.class))).thenAnswer(
- new Answer<AllocateResponse>() {
- public AllocateResponse answer(InvocationOnMock invocation)
- throws Exception {
- amc.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes,
- racks, priority));
- amc.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks,
- priority));
- amc.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null,
- priority2, 0, true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- throw new Exception();
- }
- });
- amClient.rmClient = mockRM;
- amClient.allocate(0.1f);
- } catch (Exception ioe) {
- } finally {
- amClient.rmClient = realRM;
- }
-
- assertEquals(3, amClient.release.size());
- assertEquals(6, amClient.ask.size());
-
- iterationsLeft = 3;
- // do a few iterations to ensure RM is not going send new containers
- while (iterationsLeft-- > 0) {
- // inform RM of rejection
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- // RM did not send new containers because AM does not need any
- assertEquals(0, allocResponse.getAllocatedContainers().size());
- if (allocResponse.getCompletedContainersStatuses().size() > 0) {
- for (ContainerStatus cStatus : allocResponse
- .getCompletedContainersStatuses()) {
- if (releases.contains(cStatus.getContainerId())) {
- assertEquals(cStatus.getState(), ContainerState.COMPLETE);
- assertEquals(-100, cStatus.getExitStatus());
- releases.remove(cStatus.getContainerId());
- }
- }
- }
- if (iterationsLeft > 0) {
- // sleep to make sure NM's heartbeat
- sleep(100);
- }
- }
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
- null, null);
-
- } finally {
- if (amClient != null && amClient.getServiceState() == Service.STATE
- .STARTED) {
- amClient.stop();
- }
- }
- }
-
- /**
- * Check if an AM can ask for opportunistic containers and get them.
- * @throws Exception
- */
- @Test
- public void testAMOpportunistic() throws Exception {
- // Basic container to request
- Resource capability = Resource.newInstance(1024, 1);
- Priority priority = Priority.newInstance(1);
-
- // Get the cluster topology
- List<NodeReport> nodeReports = rmClient.getNodeReports(NodeState.RUNNING);
- String node = nodeReports.get(0).getNodeId().getHost();
- String rack = nodeReports.get(0).getRackName();
- String[] nodes = new String[]{node};
- String[] racks = new String[]{rack};
-
- // Create an AM to request resources
- AMRMClient<AMRMClient.ContainerRequest> amClient = null;
- try {
- amClient = new AMRMClientImpl<AMRMClient.ContainerRequest>(client);
- amClient.init(yarnConf);
- amClient.start();
- amClient.registerApplicationMaster(NetUtils.getHostname(), 1024, "");
-
- // AM requests an opportunistic container
- ExecutionTypeRequest execTypeRequest =
- ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC, true);
- ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
- capability, nodes, racks, priority, 0, true, null, execTypeRequest);
- amClient.addContainerRequest(containerRequest);
-
- // Wait until the container is allocated
- ContainerId opportunisticContainerId = null;
- for (int i=0; i<10 && opportunisticContainerId == null; i++) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- List<Container> allocatedContainers =
- allocResponse.getAllocatedContainers();
- for (Container allocatedContainer : allocatedContainers) {
- // Check that this is the container we required
- assertEquals(ExecutionType.OPPORTUNISTIC,
- allocatedContainer.getExecutionType());
- opportunisticContainerId = allocatedContainer.getId();
- }
- sleep(100);
- }
- assertNotNull(opportunisticContainerId);
-
- // The RM sees the container as OPPORTUNISTIC
- ResourceScheduler scheduler = cluster.getResourceManager()
- .getResourceScheduler();
- RMContainer rmContainer = scheduler.getRMContainer(
- opportunisticContainerId);
- assertEquals(ExecutionType.OPPORTUNISTIC,
- rmContainer.getExecutionType());
-
- // Release the opportunistic container
- amClient.releaseAssignedContainer(opportunisticContainerId);
- // Wait for the release container to appear
- boolean released = false;
- for (int i=0; i<10 && !released; i++) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- List<ContainerStatus> completedContainers =
- allocResponse.getCompletedContainersStatuses();
- for (ContainerStatus completedContainer : completedContainers) {
- ContainerId completedContainerId =
- completedContainer.getContainerId();
- assertEquals(completedContainerId, opportunisticContainerId);
- released = true;
- }
- if (!released) {
- sleep(100);
- }
- }
- assertTrue(released);
-
- // The RM shouldn't see the container anymore
- rmContainer = scheduler.getRMContainer(opportunisticContainerId);
- assertNull(rmContainer);
-
- // Clean the AM
- amClient.unregisterApplicationMaster(
- FinalApplicationStatus.SUCCEEDED, null, null);
- } finally {
- if (amClient != null &&
- amClient.getServiceState() == Service.STATE.STARTED) {
- amClient.close();
- }
- }
- }
-
- private void sleep(int sleepTime) {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b733348d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
deleted file mode 100644
index 12c32fc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestOpportunisticContainerAllocation.java
+++ /dev/null
@@ -1,784 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.client.api.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
-import org.apache.hadoop.yarn.api.records.ExecutionType;
-import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NMToken;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.NMTokenCache;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler
- .AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-/**
- * Class that tests the allocation of OPPORTUNISTIC containers through the
- * centralized ResourceManager.
- */
-public class TestOpportunisticContainerAllocation {
- private static Configuration conf = null;
- private static MiniYARNCluster yarnCluster = null;
- private static YarnClient yarnClient = null;
- private static List<NodeReport> nodeReports = null;
- private static int nodeCount = 3;
-
- private static final int ROLLING_INTERVAL_SEC = 13;
- private static final long AM_EXPIRE_MS = 4000;
-
- private static Resource capability;
- private static ProfileCapability profileCapability;
- private static Priority priority;
- private static Priority priority2;
- private static Priority priority3;
- private static Priority priority4;
- private static String node;
- private static String rack;
- private static String[] nodes;
- private static String[] racks;
- private final static int DEFAULT_ITERATION = 3;
-
- // Per test..
- private ApplicationAttemptId attemptId = null;
- private AMRMClientImpl<AMRMClient.ContainerRequest> amClient = null;
- private long availMB;
- private int availVCores;
- private long allocMB;
- private int allocVCores;
-
- @BeforeClass
- public static void setup() throws Exception {
- // start minicluster
- conf = new YarnConfiguration();
- conf.setLong(
- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
- ROLLING_INTERVAL_SEC);
- conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, AM_EXPIRE_MS);
- conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1000);
- // set the minimum allocation so that resource decrease can go under 1024
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
- conf.setBoolean(
- YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
- conf.setInt(
- YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH, 10);
- conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
- yarnCluster =
- new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- // start rm client
- yarnClient = YarnClient.createYarnClient();
- yarnClient.init(conf);
- yarnClient.start();
-
- // get node info
- nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
-
- priority = Priority.newInstance(1);
- priority2 = Priority.newInstance(2);
- priority3 = Priority.newInstance(3);
- priority4 = Priority.newInstance(4);
- capability = Resource.newInstance(512, 1);
- profileCapability = ProfileCapability.newInstance(capability);
-
- node = nodeReports.get(0).getNodeId().getHost();
- rack = nodeReports.get(0).getRackName();
- nodes = new String[]{node};
- racks = new String[]{rack};
- }
-
- @Before
- public void startApp() throws Exception {
- // submit new app
- ApplicationSubmissionContext appContext =
- yarnClient.createApplication().getApplicationSubmissionContext();
- ApplicationId appId = appContext.getApplicationId();
- // set the application name
- appContext.setApplicationName("Test");
- // Set the priority for the application master
- Priority pri = Records.newRecord(Priority.class);
- pri.setPriority(0);
- appContext.setPriority(pri);
- // Set the queue to which this application is to be submitted in the RM
- appContext.setQueue("default");
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext(
- Collections.<String, LocalResource>emptyMap(),
- new HashMap<String, String>(), Arrays.asList("sleep", "100"),
- new HashMap<String, ByteBuffer>(), null,
- new HashMap<ApplicationAccessType, String>());
- appContext.setAMContainerSpec(amContainer);
- appContext.setResource(Resource.newInstance(1024, 1));
- // Create the request to send to the applications manager
- SubmitApplicationRequest appRequest =
- Records.newRecord(SubmitApplicationRequest.class);
- appRequest.setApplicationSubmissionContext(appContext);
- // Submit the application to the applications manager
- yarnClient.submitApplication(appContext);
-
- // wait for app to start
- RMAppAttempt appAttempt = null;
- while (true) {
- ApplicationReport appReport = yarnClient.getApplicationReport(appId);
- if (appReport.getYarnApplicationState() ==
- YarnApplicationState.ACCEPTED) {
- attemptId = appReport.getCurrentApplicationAttemptId();
- appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
- .get(attemptId.getApplicationId()).getCurrentAppAttempt();
- while (true) {
- if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
- break;
- }
- }
- break;
- }
- }
- // Just dig into the ResourceManager and get the AMRMToken just for the sake
- // of testing.
- UserGroupInformation.setLoginUser(UserGroupInformation
- .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
-
- // emulate RM setup of AMRM token in credentials by adding the token
- // *before* setting the token service
- UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
- appAttempt.getAMRMToken()
- .setService(ClientRMProxy.getAMRMTokenService(conf));
-
- // start am rm client
- amClient = (AMRMClientImpl<AMRMClient.ContainerRequest>)AMRMClient
- .createAMRMClient();
-
- //setting an instance NMTokenCache
- amClient.setNMTokenCache(new NMTokenCache());
- //asserting we are not using the singleton instance cache
- Assert.assertNotSame(NMTokenCache.getSingleton(),
- amClient.getNMTokenCache());
-
- amClient.init(conf);
- amClient.start();
-
- amClient.registerApplicationMaster("Host", 10000, "");
- }
-
- @After
- public void cancelApp() throws YarnException, IOException {
- try {
- amClient
- .unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null,
- null);
- } finally {
- if (amClient != null &&
- amClient.getServiceState() == Service.STATE.STARTED) {
- amClient.stop();
- }
- }
- yarnClient.killApplication(attemptId.getApplicationId());
- attemptId = null;
- }
-
- @AfterClass
- public static void tearDown() {
- if (yarnClient != null &&
- yarnClient.getServiceState() == Service.STATE.STARTED) {
- yarnClient.stop();
- }
- if (yarnCluster != null &&
- yarnCluster.getServiceState() == Service.STATE.STARTED) {
- yarnCluster.stop();
- }
- }
-
- @Test(timeout = 60000)
- public void testPromotionFromAcquired() throws YarnException, IOException {
- // setup container request
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- int oppContainersRequestedAny =
- amClient.getTable(0).get(priority2, ResourceRequest.ANY,
- ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
- .getNumContainers();
-
- assertEquals(1, oppContainersRequestedAny);
-
- assertEquals(1, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- // RM should allocate container within 2 calls to allocate()
- int allocatedContainerCount = 0;
- Map<ContainerId, Container> allocatedOpportContainers = new HashMap<>();
- int iterationsLeft = 50;
-
- amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
- amClient.getNMTokenCache().numberOfTokensInCache());
- HashMap<String, Token> receivedNMTokens = new HashMap<>();
-
- updateMetrics("Before Opp Allocation");
-
- while (allocatedContainerCount < oppContainersRequestedAny
- && iterationsLeft-- > 0) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- allocatedContainerCount +=
- allocResponse.getAllocatedContainers().size();
- for (Container container : allocResponse.getAllocatedContainers()) {
- if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
- allocatedOpportContainers.put(container.getId(), container);
- removeCR(container);
- }
- }
-
- for (NMToken token : allocResponse.getNMTokens()) {
- String nodeID = token.getNodeId().toString();
- receivedNMTokens.put(nodeID, token.getToken());
- }
-
- if (allocatedContainerCount < oppContainersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
- }
- }
-
- assertEquals(oppContainersRequestedAny, allocatedContainerCount);
- assertEquals(oppContainersRequestedAny, allocatedOpportContainers.size());
-
- updateMetrics("After Opp Allocation / Before Promotion");
-
- try {
- Container c = allocatedOpportContainers.values().iterator().next();
- amClient.requestContainerUpdate(
- c, UpdateContainerRequest.newInstance(c.getVersion(),
- c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
- null, ExecutionType.OPPORTUNISTIC));
- Assert.fail("Should throw Exception..");
- } catch (IllegalArgumentException e) {
- System.out.println("## " + e.getMessage());
- Assert.assertTrue(e.getMessage().contains(
- "target should be GUARANTEED and original should be OPPORTUNISTIC"));
- }
-
- Container c = allocatedOpportContainers.values().iterator().next();
- amClient.requestContainerUpdate(
- c, UpdateContainerRequest.newInstance(c.getVersion(),
- c.getId(), ContainerUpdateType.PROMOTE_EXECUTION_TYPE,
- null, ExecutionType.GUARANTEED));
- iterationsLeft = 120;
- Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
- // do a few iterations to ensure RM is not going to send new containers
- while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
- // inform RM of rejection
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- // RM did not send new containers because AM does not need any
- if (allocResponse.getUpdatedContainers() != null) {
- for (UpdatedContainer updatedContainer : allocResponse
- .getUpdatedContainers()) {
- System.out.println("Got update..");
- updatedContainers.put(updatedContainer.getContainer().getId(),
- updatedContainer);
- }
- }
- if (iterationsLeft > 0) {
- // sleep to make sure NM's heartbeat
- sleep(100);
- }
- }
-
- updateMetrics("After Promotion");
-
- assertEquals(1, updatedContainers.size());
- for (ContainerId cId : allocatedOpportContainers.keySet()) {
- Container orig = allocatedOpportContainers.get(cId);
- UpdatedContainer updatedContainer = updatedContainers.get(cId);
- assertNotNull(updatedContainer);
- assertEquals(ExecutionType.GUARANTEED,
- updatedContainer.getContainer().getExecutionType());
- assertEquals(orig.getResource(),
- updatedContainer.getContainer().getResource());
- assertEquals(orig.getNodeId(),
- updatedContainer.getContainer().getNodeId());
- assertEquals(orig.getVersion() + 1,
- updatedContainer.getContainer().getVersion());
- }
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
- amClient.ask.clear();
- }
-
- @Test(timeout = 60000)
- public void testDemotionFromAcquired() throws YarnException, IOException {
- // setup container request
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority3));
-
- int guarContainersRequestedAny = amClient.getTable(0).get(priority3,
- ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
- .remoteRequest.getNumContainers();
-
- assertEquals(1, guarContainersRequestedAny);
-
- assertEquals(1, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- // RM should allocate container within 2 calls to allocate()
- int allocatedContainerCount = 0;
- Map<ContainerId, Container> allocatedGuarContainers = new HashMap<>();
- int iterationsLeft = 50;
-
- amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
- amClient.getNMTokenCache().numberOfTokensInCache());
- HashMap<String, Token> receivedNMTokens = new HashMap<>();
-
- updateMetrics("Before Guar Allocation");
-
- while (allocatedContainerCount < guarContainersRequestedAny
- && iterationsLeft-- > 0) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- allocatedContainerCount +=
- allocResponse.getAllocatedContainers().size();
- for (Container container : allocResponse.getAllocatedContainers()) {
- if (container.getExecutionType() == ExecutionType.GUARANTEED) {
- allocatedGuarContainers.put(container.getId(), container);
- removeCR(container);
- }
- }
-
- for (NMToken token : allocResponse.getNMTokens()) {
- String nodeID = token.getNodeId().toString();
- receivedNMTokens.put(nodeID, token.getToken());
- }
-
- if (allocatedContainerCount < guarContainersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
- }
- }
-
- assertEquals(guarContainersRequestedAny, allocatedContainerCount);
- assertEquals(guarContainersRequestedAny, allocatedGuarContainers.size());
-
- updateMetrics("After Guar Allocation / Before Demotion");
-
- try {
- Container c = allocatedGuarContainers.values().iterator().next();
- amClient.requestContainerUpdate(
- c, UpdateContainerRequest.newInstance(c.getVersion(),
- c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
- null, ExecutionType.GUARANTEED));
- Assert.fail("Should throw Exception..");
- } catch (IllegalArgumentException e) {
- System.out.println("## " + e.getMessage());
- Assert.assertTrue(e.getMessage().contains(
- "target should be OPPORTUNISTIC and original should be GUARANTEED"));
- }
-
- Container c = allocatedGuarContainers.values().iterator().next();
- amClient.requestContainerUpdate(
- c, UpdateContainerRequest.newInstance(c.getVersion(),
- c.getId(), ContainerUpdateType.DEMOTE_EXECUTION_TYPE,
- null, ExecutionType.OPPORTUNISTIC));
- iterationsLeft = 120;
- Map<ContainerId, UpdatedContainer> updatedContainers = new HashMap<>();
- // do a few iterations to ensure RM is not going to send new containers
- while (iterationsLeft-- > 0 && updatedContainers.isEmpty()) {
- // inform RM of rejection
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- // RM did not send new containers because AM does not need any
- if (allocResponse.getUpdatedContainers() != null) {
- for (UpdatedContainer updatedContainer : allocResponse
- .getUpdatedContainers()) {
- System.out.println("Got update..");
- updatedContainers.put(updatedContainer.getContainer().getId(),
- updatedContainer);
- }
- }
- if (iterationsLeft > 0) {
- // sleep to make sure NM's heartbeat
- sleep(100);
- }
- }
-
- updateMetrics("After Demotion");
-
- assertEquals(1, updatedContainers.size());
- for (ContainerId cId : allocatedGuarContainers.keySet()) {
- Container orig = allocatedGuarContainers.get(cId);
- UpdatedContainer updatedContainer = updatedContainers.get(cId);
- assertNotNull(updatedContainer);
- assertEquals(ExecutionType.OPPORTUNISTIC,
- updatedContainer.getContainer().getExecutionType());
- assertEquals(orig.getResource(),
- updatedContainer.getContainer().getResource());
- assertEquals(orig.getNodeId(),
- updatedContainer.getContainer().getNodeId());
- assertEquals(orig.getVersion() + 1,
- updatedContainer.getContainer().getVersion());
- }
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
- amClient.ask.clear();
- }
-
- @Test(timeout = 60000)
- public void testMixedAllocationAndRelease() throws YarnException,
- IOException {
- // setup container request
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
-
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- int containersRequestedNode = amClient.getTable(0).get(priority,
- node, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- int containersRequestedRack = amClient.getTable(0).get(priority,
- rack, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- int containersRequestedAny = amClient.getTable(0).get(priority,
- ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
- .remoteRequest.getNumContainers();
- int oppContainersRequestedAny =
- amClient.getTable(0).get(priority2, ResourceRequest.ANY,
- ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
- .getNumContainers();
-
- assertEquals(4, containersRequestedNode);
- assertEquals(4, containersRequestedRack);
- assertEquals(4, containersRequestedAny);
- assertEquals(2, oppContainersRequestedAny);
-
- assertEquals(4, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority2, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- containersRequestedNode = amClient.getTable(0).get(priority,
- node, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- containersRequestedRack = amClient.getTable(0).get(priority,
- rack, ExecutionType.GUARANTEED, profileCapability).remoteRequest
- .getNumContainers();
- containersRequestedAny = amClient.getTable(0).get(priority,
- ResourceRequest.ANY, ExecutionType.GUARANTEED, profileCapability)
- .remoteRequest.getNumContainers();
- oppContainersRequestedAny =
- amClient.getTable(0).get(priority2, ResourceRequest.ANY,
- ExecutionType.OPPORTUNISTIC, profileCapability).remoteRequest
- .getNumContainers();
-
- assertEquals(2, containersRequestedNode);
- assertEquals(2, containersRequestedRack);
- assertEquals(2, containersRequestedAny);
- assertEquals(1, oppContainersRequestedAny);
-
- assertEquals(4, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- // RM should allocate container within 2 calls to allocate()
- int allocatedContainerCount = 0;
- int allocatedOpportContainerCount = 0;
- int iterationsLeft = 50;
- Set<ContainerId> releases = new TreeSet<>();
-
- amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
- amClient.getNMTokenCache().numberOfTokensInCache());
- HashMap<String, Token> receivedNMTokens = new HashMap<>();
-
- while (allocatedContainerCount <
- containersRequestedAny + oppContainersRequestedAny
- && iterationsLeft-- > 0) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- allocatedContainerCount +=
- allocResponse.getAllocatedContainers().size();
- for (Container container : allocResponse.getAllocatedContainers()) {
- if (container.getExecutionType() == ExecutionType.OPPORTUNISTIC) {
- allocatedOpportContainerCount++;
- }
- ContainerId rejectContainerId = container.getId();
- releases.add(rejectContainerId);
- }
-
- for (NMToken token : allocResponse.getNMTokens()) {
- String nodeID = token.getNodeId().toString();
- receivedNMTokens.put(nodeID, token.getToken());
- }
-
- if (allocatedContainerCount < containersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
- }
- }
-
- assertEquals(containersRequestedAny + oppContainersRequestedAny,
- allocatedContainerCount);
- assertEquals(oppContainersRequestedAny, allocatedOpportContainerCount);
- for (ContainerId rejectContainerId : releases) {
- amClient.releaseAssignedContainer(rejectContainerId);
- }
- assertEquals(3, amClient.release.size());
- assertEquals(0, amClient.ask.size());
-
- // need to tell the AMRMClient that we don't need these resources anymore
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority));
- amClient.removeContainerRequest(
- new AMRMClient.ContainerRequest(capability, nodes, racks, priority2, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- assertEquals(4, amClient.ask.size());
-
- iterationsLeft = 3;
- // do a few iterations to ensure RM is not going to send new containers
- while (iterationsLeft-- > 0) {
- // inform RM of rejection
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- // RM did not send new containers because AM does not need any
- assertEquals(0, allocResponse.getAllocatedContainers().size());
- if (allocResponse.getCompletedContainersStatuses().size() > 0) {
- for (ContainerStatus cStatus : allocResponse
- .getCompletedContainersStatuses()) {
- if (releases.contains(cStatus.getContainerId())) {
- assertEquals(cStatus.getState(), ContainerState.COMPLETE);
- assertEquals(-100, cStatus.getExitStatus());
- releases.remove(cStatus.getContainerId());
- }
- }
- }
- if (iterationsLeft > 0) {
- // sleep to make sure NM's heartbeat
- sleep(100);
- }
- }
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
- }
-
- /**
- * Tests allocation with requests comprising only opportunistic containers.
- */
- @Test(timeout = 60000)
- public void testOpportunisticAllocation() throws YarnException, IOException {
- // setup container request
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
- amClient.addContainerRequest(
- new AMRMClient.ContainerRequest(capability, null, null, priority3, 0,
- true, null,
- ExecutionTypeRequest.newInstance(
- ExecutionType.OPPORTUNISTIC, true)));
-
- int oppContainersRequestedAny = amClient.getTable(0)
- .get(priority3, ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC,
- profileCapability).remoteRequest.getNumContainers();
-
- assertEquals(2, oppContainersRequestedAny);
-
- assertEquals(1, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- // RM should allocate container within 2 calls to allocate()
- int allocatedContainerCount = 0;
- int iterationsLeft = 10;
- Set<ContainerId> releases = new TreeSet<>();
-
- amClient.getNMTokenCache().clearCache();
- Assert.assertEquals(0,
- amClient.getNMTokenCache().numberOfTokensInCache());
- HashMap<String, Token> receivedNMTokens = new HashMap<>();
-
- while (allocatedContainerCount < oppContainersRequestedAny
- && iterationsLeft-- > 0) {
- AllocateResponse allocResponse = amClient.allocate(0.1f);
- assertEquals(0, amClient.ask.size());
- assertEquals(0, amClient.release.size());
-
- for (Container container : allocResponse.getAllocatedContainers()) {
- allocatedContainerCount++;
- ContainerId rejectContainerId = container.getId();
- releases.add(rejectContainerId);
- }
-
- for (NMToken token : allocResponse.getNMTokens()) {
- String nodeID = token.getNodeId().toString();
- receivedNMTokens.put(nodeID, token.getToken());
- }
-
- if (allocatedContainerCount < oppContainersRequestedAny) {
- // sleep to let NM's heartbeat to RM and trigger allocations
- sleep(100);
- }
- }
-
- assertEquals(oppContainersRequestedAny, allocatedContainerCount);
- assertEquals(1, receivedNMTokens.values().size());
- }
-
- private void removeCR(Container container) {
- List<? extends Collection<AMRMClient.ContainerRequest>>
- matchingRequests = amClient.getMatchingRequests(container
- .getPriority(),
- ResourceRequest.ANY, ExecutionType.OPPORTUNISTIC,
- container.getResource());
- Set<AMRMClient.ContainerRequest> toRemove = new HashSet<>();
- for (Collection<AMRMClient.ContainerRequest> rc : matchingRequests) {
- for (AMRMClient.ContainerRequest cr : rc) {
- toRemove.add(cr);
- }
- }
- for (AMRMClient.ContainerRequest cr : toRemove) {
- amClient.removeContainerRequest(cr);
- }
- }
-
- private void updateMetrics(String msg) {
- AbstractYarnScheduler scheduler =
- (AbstractYarnScheduler)yarnCluster.getResourceManager()
- .getResourceScheduler();
- availMB = scheduler.getRootQueueMetrics().getAvailableMB();
- availVCores = scheduler.getRootQueueMetrics().getAvailableVirtualCores();
- allocMB = scheduler.getRootQueueMetrics().getAllocatedMB();
- allocVCores = scheduler.getRootQueueMetrics().getAllocatedVirtualCores();
- System.out.println("## METRICS (" + msg + ")==>");
- System.out.println(" : availMB=" + availMB + ", " +
- "availVCores=" +availVCores + ", " +
- "allocMB=" + allocMB + ", " +
- "allocVCores=" + allocVCores + ", ");
- System.out.println("<== ##");
- }
-
- private void sleep(int sleepTime) {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-7251. Misc changes to
YARN-5734
Posted by jh...@apache.org.
YARN-7251. Misc changes to YARN-5734
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a5d3f8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a5d3f8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a5d3f8c
Branch: refs/heads/YARN-5734
Commit: 3a5d3f8c7b84026b253a487f63014ff05f0e0a05
Parents: 3e73e0e
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Sep 25 17:52:40 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:54 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 3 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 33 ++++++++++++++-
.../hadoop/yarn/client/cli/SchedConfCLI.java | 42 ++++++++++++++------
.../yarn/client/cli/TestSchedConfCLI.java | 20 +++++-----
.../hadoop/yarn/webapp/dao/QueueConfigInfo.java | 2 +-
.../yarn/webapp/dao/SchedConfUpdateInfo.java | 2 +-
.../src/main/resources/yarn-default.xml | 15 ++++---
.../scheduler/capacity/CapacityScheduler.java | 12 +++---
.../CapacitySchedulerConfiguration.java | 12 ------
.../conf/MutableCSConfigurationProvider.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
.../resourcemanager/TestRMAdminService.java | 4 +-
.../conf/TestLeveldbConfigurationStore.java | 2 -
.../TestMutableCSConfigurationProvider.java | 3 ++
.../capacity/conf/TestZKConfigurationStore.java | 2 -
.../TestRMWebServicesConfigurationMutation.java | 28 ++++++-------
.../src/site/markdown/CapacityScheduler.md | 11 ++---
.../src/site/markdown/ResourceManagerRest.md | 40 ++++++++++---------
.../src/site/markdown/YarnCommands.md | 12 +++---
20 files changed, 146 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 0063596..0f19989 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,6 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
hadoop_add_subcommand "rmadmin" admin "admin tools"
hadoop_add_subcommand "router" daemon "run the Router daemon"
+ hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
@@ -142,7 +143,7 @@ function yarncmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
;;
- schedconf)
+ schedulerconf)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
;;
scmadmin)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 7ec9848..fed3d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -295,7 +295,7 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
-:schedconf
+:schedulerconf
set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
@@ -345,6 +345,7 @@ goto :eof
@echo node prints node report(s)
@echo queue prints queue information
@echo logs dump container logs
+ @echo schedulerconf updates scheduler configuration
@echo classpath prints the class path needed to get the
@echo Hadoop jar and the required libraries
@echo daemonlog get/set the log level for each daemon
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8809a7c..ac8ff12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -674,33 +674,64 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ @Private
+ @Unstable
public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
YARN_PREFIX + "scheduler.configuration.store.class";
+ @Private
+ @Unstable
+ public static final String FILE_CONFIGURATION_STORE = "file";
+ @Private
+ @Unstable
public static final String MEMORY_CONFIGURATION_STORE = "memory";
+ @Private
+ @Unstable
public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ @Private
+ @Unstable
public static final String ZK_CONFIGURATION_STORE = "zk";
+ @Private
+ @Unstable
public static final String DEFAULT_CONFIGURATION_STORE =
- MEMORY_CONFIGURATION_STORE;
+ FILE_CONFIGURATION_STORE;
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+ "scheduler.configuration.leveldb-store.path";
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
YARN_PREFIX
+ "scheduler.configuration.leveldb-store.compaction-interval-secs";
+ @Private
+ @Unstable
public static final long
DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_MAX_LOGS =
YARN_PREFIX + "scheduler.configuration.store.max-logs";
+ @Private
+ @Unstable
public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
+ @Private
+ @Unstable
public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
/** Parent znode path under which ZKConfigurationStore will create znodes. */
+ @Private
+ @Unstable
public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX
+ "scheduler.configuration.zk-store.parent-path";
+ @Private
+ @Unstable
public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH =
"/confstore";
+ @Private
+ @Unstable
public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index e17062e..11bfdd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -27,7 +27,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -48,7 +48,7 @@ import java.util.Map;
* CLI for modifying scheduler configuration.
*/
@Public
-@Evolving
+@Unstable
public class SchedConfCLI extends Configured implements Tool {
private static final String ADD_QUEUES_OPTION = "addQueues";
@@ -135,7 +135,7 @@ public class SchedConfCLI extends Configured implements Tool {
WebResource webResource = webServiceClient.resource(WebAppUtils.
getRMWebAppURLWithScheme(getConf()));
ClientResponse response = webResource.path("ws").path("v1").path("cluster")
- .path("sched-conf").accept(MediaType.APPLICATION_JSON)
+ .path("scheduler-conf").accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
@@ -170,7 +170,7 @@ public class SchedConfCLI extends Configured implements Tool {
if (args == null) {
return;
}
- List<String> queuesToRemove = Arrays.asList(args.split(","));
+ List<String> queuesToRemove = Arrays.asList(args.split(";"));
updateInfo.setRemoveQueueInfo(new ArrayList<>(queuesToRemove));
}
@@ -199,11 +199,14 @@ public class SchedConfCLI extends Configured implements Tool {
}
private QueueConfigInfo getQueueConfigInfo(String arg) {
- String[] queueArgs = arg.split(",");
- String queuePath = queueArgs[0];
+ String[] args = arg.split(":");
+ String queuePath = args[0];
Map<String, String> queueConfigs = new HashMap<>();
- for (int i = 1; i < queueArgs.length; ++i) {
- putKeyValuePair(queueConfigs, queueArgs[i]);
+ if (args.length > 1) {
+ String[] queueArgs = args[1].split(",");
+ for (int i = 0; i < queueArgs.length; ++i) {
+ putKeyValuePair(queueConfigs, queueArgs[i]);
+ }
}
return new QueueConfigInfo(queuePath, queueConfigs);
}
@@ -228,11 +231,24 @@ public class SchedConfCLI extends Configured implements Tool {
}
private void printUsage() {
- System.out.println("yarn schedconf [-add queueAddPath1,confKey1=confVal1,"
- + "confKey2=confVal2;queueAddPath2,confKey3=confVal3] "
- + "[-remove queueRemovePath1,queueRemovePath2] "
- + "[-update queueUpdatePath1,confKey1=confVal1] "
+ System.out.println("yarn schedulerconf [-add "
+ + "\"queueAddPath1:confKey1=confVal1,confKey2=confVal2;"
+ + "queueAddPath2:confKey3=confVal3\"] "
+ + "[-remove \"queueRemovePath1;queueRemovePath2\"] "
+ + "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
+ "[-global globalConfKey1=globalConfVal1,"
- + "globalConfKey2=globalConfVal2]");
+ + "globalConfKey2=globalConfVal2]\n"
+ + "Example (adding queues): yarn schedulerconf -add "
+ + "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ + "maximum-capacity=0\"\n"
+ + "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;"
+ + "root.a.a2\"\n"
+ + "Example (updating queues): yarn schedulerconf -update \"root.a.a1"
+ + ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75,"
+ + "maximum-capacity=75\"\n"
+ + "Example (global scheduler update): yarn schedulerconf "
+ + "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ + "Note: This is an alpha feature, the syntax/options are subject to "
+ + "change, please run at your own risk.");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index d2f0639..5364e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -61,20 +61,20 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testInvalidConf() throws Exception {
// conf pair with no key should be invalid
- int exitCode = cli.run(new String[] {"-add", "root.a,=confVal"});
+ int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-update", "root.a,=confVal"});
+ exitCode = cli.run(new String[] {"-update", "root.a:=confVal"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-add", "root.a,confKey=confVal=conf"});
+ exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
- exitCode = cli.run(new String[] {"-update", "root.a,confKey=confVal=c"});
+ exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
@@ -83,8 +83,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testAddQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.addQueues("root.a,a1=aVal1,a2=aVal2," +
- "a3=", schedUpdateInfo);
+ cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
assertEquals("root.a", addInfo.getQueue());
Map<String, String> params = addInfo.getParams();
@@ -94,7 +93,7 @@ public class TestSchedConfCLI {
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo();
- cli.addQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getAddQueueInfo().size());
QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0);
assertEquals("root.b", bAddInfo.getQueue());
@@ -111,7 +110,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testRemoveQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.removeQueues("root.a,root.b,root.c.c1", schedUpdateInfo);
+ cli.removeQueues("root.a;root.b;root.c.c1", schedUpdateInfo);
List<String> removeInfo = schedUpdateInfo.getRemoveQueueInfo();
assertEquals(3, removeInfo.size());
assertEquals("root.a", removeInfo.get(0));
@@ -122,8 +121,7 @@ public class TestSchedConfCLI {
@Test(timeout = 10000)
public void testUpdateQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
- cli.updateQueues("root.a,a1=aVal1,a2=aVal2," +
- "a3=", schedUpdateInfo);
+ cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
assertEquals("root.a", updateInfo.getQueue());
Map<String, String> params = updateInfo.getParams();
@@ -133,7 +131,7 @@ public class TestSchedConfCLI {
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo();
- cli.updateQueues("root.b,b1=bVal1;root.c,c1=cVal1", schedUpdateInfo);
+ cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size());
QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
assertEquals("root.b", bUpdateInfo.getQueue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
index d1d91c2..6d4e0cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/QueueConfigInfo.java
@@ -34,7 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueConfigInfo {
- @XmlElement(name = "queueName")
+ @XmlElement(name = "queue-name")
private String queue;
private HashMap<String, String> params = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
index bb84096..4546291 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/SchedConfUpdateInfo.java
@@ -31,7 +31,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* Information for making scheduler configuration changes (supports adding,
* removing, or updating a queue, as well as global scheduler conf changes).
*/
-@XmlRootElement(name = "schedConf")
+@XmlRootElement(name = "sched-conf")
@XmlAccessorType(XmlAccessType.FIELD)
public class SchedConfUpdateInfo {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4516edf..421bccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3369,14 +3369,17 @@
<property>
<description>
- The type of configuration store to use for storing scheduler
- configurations, if using a mutable configuration provider.
- Keywords such as "memory" map to certain configuration store
- implementations. If keyword is not found, try to load this
- value as a class.
+ The type of configuration store to use for scheduler configurations.
+ Default is "file", which uses file based capacity-scheduler.xml to
+ retrieve and change scheduler configuration. To enable API based
+ scheduler configuration, use either "memory" (in memory storage, no
+ persistence across restarts), "leveldb" (leveldb based storage), or
+ "zk" (zookeeper based storage). API based configuration is only useful
+ when using a scheduler which supports mutable configuration. Currently
+ only capacity scheduler supports this.
</description>
<name>yarn.scheduler.configuration.store.class</name>
- <value>memory</value>
+ <value>file</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index de95179..d91aa55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -296,18 +296,20 @@ public class CapacityScheduler extends
try {
writeLock.lock();
String confProviderStr = configuration.get(
- CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
switch (confProviderStr) {
- case CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER:
+ case YarnConfiguration.FILE_CONFIGURATION_STORE:
this.csConfProvider =
new FileBasedCSConfigurationProvider(rmContext);
break;
- case CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER:
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
break;
default:
- throw new IOException("Invalid CS configuration provider: " +
+ throw new IOException("Invalid configuration store class: " +
confProviderStr);
}
this.csConfProvider.init(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 40cb893..3a519ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -315,18 +315,6 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
-
- public static final String CS_CONF_PROVIDER = PREFIX
- + "configuration.provider";
-
- @Private
- public static final String FILE_CS_CONF_PROVIDER = "file";
-
- @Private
- public static final String STORE_CS_CONF_PROVIDER = "store";
-
- @Private
- public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ccadf76..40a19a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -66,7 +66,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
public void init(Configuration config) throws IOException {
String store = config.get(
YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
- YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
switch (store) {
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
this.confStore = new InMemoryConfigurationStore();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d264c10..e6a0cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2461,7 +2461,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@PUT
- @Path("/sched-conf")
+ @Path("/scheduler-conf")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 620ca58..03fc081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -199,8 +199,8 @@ public class TestRMAdminService {
@Test
public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() {
- configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ configuration.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
try {
rm = new MockRM(configuration);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 779208a..324cbee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -59,8 +59,6 @@ public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
public void setUp() throws Exception {
super.setUp();
FileUtil.fullyDelete(TEST_DIR);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.LEVELDB_CONFIGURATION_STORE);
conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 9b080cd..5d43ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -80,6 +81,8 @@ public class TestMutableCSConfigurationProvider {
@Test
public void testInMemoryBackedProvider() throws Exception {
Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
.get("yarn.scheduler.capacity.root.a.goodKey"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 3cfa8da..355f741 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -199,8 +199,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
YarnConfiguration.ZK_CONFIGURATION_STORE);
conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 26ef1b7..3d28f12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -88,8 +88,8 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
- conf.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
- CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
try {
if (CONF_FILE.exists()) {
@@ -179,7 +179,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getAddQueueInfo().add(d);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -214,7 +214,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -240,7 +240,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getRemoveQueueInfo().add("root.a.a2");
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -265,7 +265,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getRemoveQueueInfo().add("root.c");
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -296,7 +296,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(b);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -328,7 +328,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(configInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -356,7 +356,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -391,7 +391,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
0.001f);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -409,7 +409,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -439,7 +439,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -464,7 +464,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
@@ -479,7 +479,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
// Unset maximum-applications. Should be set to default.
response =
r.path("ws").path("v1").path("cluster")
- .path("sched-conf").queryParam("user.name", userName)
+ .path("scheduler-conf").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index d70f891..17a6ab9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -295,7 +295,7 @@ The `ReservationSystem` is integrated with the `CapacityScheduler` queue hierach
Changing Queue Configuration
----------------------------
-Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API.
+Changing queue/scheduler properties and adding/removing queues can be done in two ways, via file or via API. This behavior can be changed via `yarn.scheduler.configuration.store.class` in yarn-site.xml. Possible values are *file*, which allows modifying properties via file; *memory*, which allows modifying properties via API, but does not persist changes across restart; *leveldb*, which allows modifying properties via API and stores changes in leveldb backing store; and *zk*, which allows modifying properties via API and stores changes in zookeeper backing store. The default value is *file*.
### Changing queue configuration via file
@@ -308,16 +308,17 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw
Editing by API uses a backing store for the scheduler configuration. To enable this, the following parameters can be configured in yarn-site.xml.
+ **Note:** This feature is in alpha phase and is subject to change.
+
| Property | Description |
|:---- |:---- |
- | `yarn.scheduler.capacity.configuration.provider` | The type of configuration provider to use for capacity scheduler. To enable changing queue configuration via API, this should be set to *store*. Default value is *file*, which disables the API and reverts back to changing queue configuration via file. |
- | `yarn.scheduler.configuration.store.class` | The type of backing store to use. Default value is *memory*, which stores the scheduler configuration in memory (and does not persist configuration changes across restarts). Other values are *leveldb* (using a leveldb-based implementation), and *zk* (using a zookeeper-based implementation). |
+ | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). |
| `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
| `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
| `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
| `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
| `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
- **Note:** When enabling backing store for scheduler configuration, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
+ **Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
- See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedconf) for examples on how to change scheduler configuration via command line.
+ See the [YARN Resource Manager REST API](ResourceManagerRest.html#Scheduler_Configuration_Mutation_API) for examples on how to change scheduler configuration via REST, and [YARN Commands Reference](YarnCommands.html#schedulerconf) for examples on how to change scheduler configuration via command line.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index 025e9a7..f3a1907 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -4436,16 +4436,18 @@ Scheduler Configuration Mutation API
The scheduler configuration mutation API provides a way to modify scheduler/queue configuration and queue hierarchy.
+Please note that this feature is currently in the alpha stage and is subject to change.
+
### URI
- * http://rm-http-address:port/ws/v1/cluster/sched-conf
+ * http://rm-http-address:port/ws/v1/cluster/scheduler-conf
### HTTP Operations Supported
* PUT
-### Elements of the *schedConf* object
+### Elements of the *sched-conf* object
| Item | Data Type | Description |
|:---- |:---- |:---- |
@@ -4466,7 +4468,7 @@ Request for updating queue configurations.
| Item | Data Type | Description |
|:---- |:---- |:---- |
-| queueName | string | Full path name of the queue to update |
+| queue-name | string | Full path name of the queue to update |
| params | map | A map of key value configuration pairs to update for this queue |
Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example sets *root.default*'s maximum applications to 100 and its minimum user limit percent to 10.
@@ -4475,12 +4477,12 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<update-queue>
- <queueName>root.default</queueName>
+ <queue-name>root.default</queue-name>
<params>
<entry>
<key>maximum-applications</key>
@@ -4492,7 +4494,7 @@ HTTP Request:
</entry>
</params>
</update-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4511,7 +4513,7 @@ Request for adding queues/updating queue configurations.
| Item | Data Type | Description |
|:---- |:---- |:---- |
-| queueName | string | Full path name of the queue to add |
+| queue-name | string | Full path name of the queue to add |
| params | map | A map of key value configuration pairs to set for this queue |
Assuming we are using the capacity scheduler and the current queue configuration is a single queue *root.default*, this example adds a queue *root.a* with capacity/maximum-capacity 10, and adjusts *root.default*'s capacity/maximum-capacity to 90. (More complex examples include adding a queue whose parent is also being added in the same request, or adding multiple sibling queues.)
@@ -4520,12 +4522,12 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<add-queue>
- <queueName>root.a</queueName>
+ <queue-name>root.a</queue-name>
<params>
<entry>
<key>capacity</key>
@@ -4538,7 +4540,7 @@ HTTP Request:
</params>
</add-queue>
<update-queue>
- <queueName>root.default</queueName>
+ <queue-name>root.default</queue-name>
<params>
<entry>
<key>capacity</key>
@@ -4550,7 +4552,7 @@ HTTP Request:
</entry>
</params>
</update-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4573,13 +4575,13 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<remove-queue>root.a</remove-queue>
<remove-queue>root.b</remove-queue>
- </schedConf>
+ </sched-conf>
```
@@ -4597,17 +4599,17 @@ HTTP Request:
```xml
Accept: application/xml
- PUT http://rm-http-address:port/ws/v1/cluster/sched-conf
+ PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf
Content-Type: application/xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
- <schedConf>
+ <sched-conf>
<global-updates>
<entry>
<key>yarn.scheduler.capacity.queue-mappings-override.enable</key>
<value>true</value>
</entry>
</global-updates>
- </schedConf>
+ </sched-conf>
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a5d3f8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index 2fc65bc..fa33b8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -237,18 +237,18 @@ Usage:
Runs ResourceManager admin client
-### schedconf
+### schedulerconf
-Usage: `yarn schedconf [options]`
+Usage: `yarn schedulerconf [options]`
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
-| -add <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
-| -remove <queuePath1,queuePath2> | Comma-separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
-| -update <queuePath1,key1=val1,key2=val2;queuePath2,key3=val3> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
+| -add <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues to add and their queue configurations. This example adds queue "queuePath1" (a full path name), which has queue configurations key1=val1 and key2=val2. It also adds queue "queuePath2", which has queue configuration key3=val3. |
+| -remove <"queuePath1;queuePath2"> | Semicolon separated queues to remove. This example removes queuePath1 and queuePath2 queues (full path names). **Note:** Queues must be put into `STOPPED` state before they are deleted. |
+| -update <"queuePath1:key1=val1,key2=val2;queuePath2:key3=val3"> | Semicolon separated values of queues whose configurations should be updated. This example sets key1=val1 and key2=val2 for queue configuration of queuePath1 (full path name), and sets key3=val3 for queue configuration of queuePath2. |
| -global <key1=val1,key2=val2> | Update scheduler global configurations. This example sets key1=val1 and key2=val2 for scheduler's global configuration. |
-Updates scheduler configuration
+Updates scheduler configuration. Note, this feature is in alpha phase and is subject to change.
### scmadmin
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: Add 3.0.0-beta1 CHANGES and
RELEASENOTES.
Posted by jh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b34b3ff9/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md
new file mode 100644
index 0000000..c799933
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md
@@ -0,0 +1,377 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# Apache Hadoop 3.0.0-beta1 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HADOOP-14535](https://issues.apache.org/jira/browse/HADOOP-14535) | *Major* | **wasb: implement high-performance random access and seek of block blobs**
+
+Random access and seek improvements for the wasb:// (Azure) file system.
+
+
+---
+
+* [YARN-6798](https://issues.apache.org/jira/browse/YARN-6798) | *Major* | **Fix NM startup failure with old state store due to version mismatch**
+
+<!-- markdown -->
+
+This fixes the LevelDB state store for the NodeManager. As of this patch, the state store versions now correspond to the following table.
+
+* Previous Patch: YARN-5049
+ * LevelDB Key: queued
+ * Hadoop Versions: 2.9.0, 3.0.0-alpha1
+ * Corresponding LevelDB Version: 1.2
+* Previous Patch: YARN-6127
+ * LevelDB Key: AMRMProxy/NextMasterKey
+ * Hadoop Versions: 2.9.0, 3.0.0-alpha4
+ * Corresponding LevelDB Version: 1.1
+
+
+---
+
+* [HADOOP-14539](https://issues.apache.org/jira/browse/HADOOP-14539) | *Major* | **Move commons logging APIs over to slf4j in hadoop-common**
+
+In Hadoop common, fatal log level is changed to error because slf4j API does not support fatal log level.
+
+
+---
+
+* [HADOOP-14518](https://issues.apache.org/jira/browse/HADOOP-14518) | *Minor* | **Customize User-Agent header sent in HTTP/HTTPS requests by WASB.**
+
+WASB now includes the current Apache Hadoop version in the User-Agent string passed to Azure Blob service. Users also may include optional additional information to identify their application. See the documentation of configuration property fs.wasb.user.agent.id for further details.
+
+
+---
+
+* [HADOOP-11875](https://issues.apache.org/jira/browse/HADOOP-11875) | *Major* | **[JDK9] Add a second copy of Hamlet without \_ as a one-character identifier**
+
+Added org.apache.hadoop.yarn.webapp.hamlet2 package without \_ as a one-character identifier. Please use this package instead of org.apache.hadoop.yarn.webapp.hamlet.
+
+
+---
+
+* [HDFS-12206](https://issues.apache.org/jira/browse/HDFS-12206) | *Major* | **Rename the split EC / replicated block metrics**
+
+The metrics and MBeans introduced in HDFS-10999 have been renamed for brevity and clarity.
+
+
+---
+
+* [HADOOP-13595](https://issues.apache.org/jira/browse/HADOOP-13595) | *Blocker* | **Rework hadoop\_usage to be broken up by clients/daemons/etc.**
+
+This patch changes how usage output is generated to now require a sub-command type. This allows users to see who the intended audience for a command is or it is a daemon.
+
+
+---
+
+* [HDFS-6984](https://issues.apache.org/jira/browse/HDFS-6984) | *Major* | **Serialize FileStatus via protobuf**
+
+FileStatus and FsPermission Writable serialization is deprecated and its implementation (incompatibly) replaced with protocol buffers. The FsPermissionProto record moved from hdfs.proto to acl.proto. HdfsFileStatus is now a subtype of FileStatus. FsPermissionExtension with its associated flags for ACLs, encryption, and erasure coding has been deprecated; users should query these attributes on the FileStatus object directly. The FsPermission instance in AclStatus no longer retains or reports these extended attributes (likely unused).
+
+
+---
+
+* [HADOOP-14722](https://issues.apache.org/jira/browse/HADOOP-14722) | *Major* | **Azure: BlockBlobInputStream position incorrect after seek**
+
+Bug fix to Azure Filesystem related to HADOOP-14535.
+
+
+---
+
+* [YARN-6961](https://issues.apache.org/jira/browse/YARN-6961) | *Minor* | **Remove commons-logging dependency from hadoop-yarn-server-applicationhistoryservice module**
+
+commons-logging dependency was removed from hadoop-yarn-server-applicationhistoryservice. If you rely on the transitive commons-logging dependency, please define the dependency explicitly.
+
+
+---
+
+* [HADOOP-14680](https://issues.apache.org/jira/browse/HADOOP-14680) | *Minor* | **Azure: IndexOutOfBoundsException in BlockBlobInputStream**
+
+Bug fix to Azure Filesystem related to HADOOP-14535
+
+
+---
+
+* [HDFS-10326](https://issues.apache.org/jira/browse/HDFS-10326) | *Major* | **Disable setting tcp socket send/receive buffers for write pipelines**
+
+The size of the TCP socket buffers are no longer hardcoded by default. Instead the OS now will automatically tune the size for the buffer.
+
+
+---
+
+* [HDFS-11957](https://issues.apache.org/jira/browse/HDFS-11957) | *Major* | **Enable POSIX ACL inheritance by default**
+
+<!-- markdown -->
+HDFS-6962 introduced POSIX ACL inheritance feature but it is disable by
+default. Now enable the feature by default. Please be aware any code
+expecting the old ACL inheritance behavior will have to be updated.
+Please see the HDFS Permissions Guide for further details.
+
+
+---
+
+* [MAPREDUCE-6870](https://issues.apache.org/jira/browse/MAPREDUCE-6870) | *Major* | **Add configuration for MR job to finish when all reducers are complete (even with unfinished mappers)**
+
+Enables mapreduce.job.finish-when-all-reducers-done by default. With this enabled, a MapReduce job will complete as soon as all of its reducers are complete, even if some mappers are still running. This can occur if a mapper was relaunched after node failure but the relaunched task's output is not actually needed. Previously the job would wait for all mappers to complete.
+
+
+---
+
+* [HADOOP-14260](https://issues.apache.org/jira/browse/HADOOP-14260) | *Major* | **Configuration.dumpConfiguration should redact sensitive information**
+
+<!-- markdown -->
+Configuration.dumpConfiguration no longer prints out the clear text values for the sensitive keys listed in `hadoop.security.sensitive-config-keys`. Callers can override the default list of sensitive keys either to redact more keys or print the clear text values for a few extra keys for debugging purpose.
+
+
+---
+
+* [HDFS-12221](https://issues.apache.org/jira/browse/HDFS-12221) | *Major* | **Replace xerces in XmlEditsVisitor**
+
+New patch with changes in maven dependency. (removed apache xcerces as dependency)
+
+
+---
+
+* [HADOOP-14726](https://issues.apache.org/jira/browse/HADOOP-14726) | *Minor* | **Mark FileStatus::isDir as final**
+
+The deprecated FileStatus::isDir method has been marked as final. FileSystems should override FileStatus::isDirectory
+
+
+---
+
+* [HADOOP-14660](https://issues.apache.org/jira/browse/HADOOP-14660) | *Major* | **wasb: improve throughput by 34% when account limit exceeded**
+
+Up to 34% throughput improvement for the wasb:// (Azure) file system when fs.azure.selfthrottling.enable is false fs.azure.autothrottling.enable is true.
+
+
+---
+
+* [HADOOP-14769](https://issues.apache.org/jira/browse/HADOOP-14769) | *Major* | **WASB: delete recursive should not fail if a file is deleted**
+
+Recursive directory delete improvement for the wasb filesystem.
+
+
+---
+
+* [YARN-6959](https://issues.apache.org/jira/browse/YARN-6959) | *Major* | **RM may allocate wrong AM Container for new attempt**
+
+ResourceManager will now record ResourceRequests from different attempts into different objects.
+
+
+---
+
+* [HDFS-12303](https://issues.apache.org/jira/browse/HDFS-12303) | *Blocker* | **Change default EC cell size to 1MB for better performance**
+
+The cell size of the provided HDFS erasure coding policies has been changed from 64k to 1024k for better performance. The policy names have all been changed accordingly, i.e. RS-6-3.1024k.
+
+
+---
+
+* [HDFS-12258](https://issues.apache.org/jira/browse/HDFS-12258) | *Major* | **ec -listPolicies should list all policies in system, no matter it's enabled or disabled**
+
+<!-- markdown -->
+
+`hdfs ec -listPolicies` now lists enabled, disabled, and removed policies, rather than just enabled policies.
+
+
+---
+
+* [MAPREDUCE-6892](https://issues.apache.org/jira/browse/MAPREDUCE-6892) | *Major* | **Issues with the count of failed/killed tasks in the jhist file**
+
+This adds some new job counters, the number of failed MAP/REDUCE tasks and the number of killed MAP/REDUCE tasks.
+
+
+---
+
+* [YARN-5355](https://issues.apache.org/jira/browse/YARN-5355) | *Critical* | **YARN Timeline Service v.2: alpha 2**
+
+We are releasing the alpha2 version of a major revision of YARN Timeline Service: v.2. YARN Timeline Service v.2 addresses two major challenges: improving scalability and reliability of Timeline Service, and enhancing usability by introducing flows and aggregation.
+
+YARN Timeline Service v.2 alpha1 was introduced in 3.0.0-alpha1 via YARN-2928.
+
+YARN Timeline Service v.2 alpha2 is now being provided so that users and developers can test it and provide feedback and suggestions for making it a ready replacement for Timeline Service v.1.x. Security is provided via Kerberos Authentication and delegation tokens. There is also a simple read level authorization provided via whitelists.
+
+Some of the notable improvements since alpha-1 are:
+- Security via Kerberos Authentication and delegation tokens
+- Read side simple authorization via whitelist
+- Client configurable entity sort ordering
+- New REST APIs for apps, app attempts, containers, fetching metrics by timerange, pagination, sub-app entities
+- Support for storing sub-application entities (entities that exist outside the scope of an application)
+- Configurable TTLs (time-to-live) for tables, configurable table prefixes, configurable hbase cluster
+- Flow level aggregations done as dynamic (table level) coprocessors
+- Uses latest stable HBase release 1.2.6
+
+More details are available in the [YARN Timeline Service v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html) documentation.
+
+
+---
+
+* [HADOOP-14364](https://issues.apache.org/jira/browse/HADOOP-14364) | *Major* | **refresh changelog/release notes with newer Apache Yetus build**
+
+Additionally, this patch updates maven-site-plugin to 3.6 and doxia-module-markdown to 1.8-SNAPSHOT to work around problems with unknown schemas in URLs in markdown formatted content.
+
+
+---
+
+* [HADOOP-13345](https://issues.apache.org/jira/browse/HADOOP-13345) | *Major* | **S3Guard: Improved Consistency for S3A**
+
+S3Guard (pronounced see-guard) is a new feature for the S3A connector to Amazon S3, which uses DynamoDB for a high performance and consistent metadata repository. Essentially: S3Guard caches directory information, so your S3A clients get faster lookups and resilience to inconsistency between S3 list operations and the status of objects. When files are created, with S3Guard, they'll always be found.
+
+S3Guard does not address update consistency: if a file is updated, while the directory information will be updated, calling open() on the path may still return the old data. Similarly, deleted objects may also potentially be opened.
+
+Please consult the S3Guard documentation in the Amazon S3 section of our documentation.
+
+Note: part of this update includes moving to a new version of the AWS SDK 1.11, one which includes the Dynamo DB client and its a shaded version of Jackson 2. The large aws-sdk-bundle JAR is needed to use the S3A client with or without S3Guard enabled. The good news: because Jackson is shaded, there will be no conflict between any Jackson version used in your application and that which the AWS SDK needs.
+
+
+---
+
+* [HADOOP-14414](https://issues.apache.org/jira/browse/HADOOP-14414) | *Minor* | **Calling maven-site-plugin directly for docs profile is unnecessary**
+
+maven-site-plugin is no longer called directly at package phase.
+
+
+---
+
+* [HDFS-12300](https://issues.apache.org/jira/browse/HDFS-12300) | *Major* | **Audit-log delegation token related operations**
+
+<!-- markdown -->
+
+NameNode now audit-logs `getDelegationToken`, `renewDelegationToken`, `cancelDelegationToken`.
+
+
+---
+
+* [HDFS-12218](https://issues.apache.org/jira/browse/HDFS-12218) | *Blocker* | **Rename split EC / replicated block metrics in BlockManager**
+
+This renames ClientProtocol#getECBlockGroupsStats to ClientProtocol#getEcBlockGroupStats and ClientProtocol#getBlockStats to ClientProtocol#getReplicatedBlockStats. The return-type classes have also been similarly renamed. Their fields have also been renamed to drop the "stats" suffix.
+
+Additionally, ECBlockGroupStats#pendingDeletionBlockGroups has been renamed to ECBlockGroupStats#pendingDeletionBlocks, to clarify that this is the number of blocks, not block groups, pending deletion. The corresponding NameNode metric has also been renamed to PendingDeletionECBlocks.
+
+
+---
+
+* [HADOOP-13421](https://issues.apache.org/jira/browse/HADOOP-13421) | *Minor* | **Switch to v2 of the S3 List Objects API in S3A**
+
+S3A now defaults to using the "v2" S3 list API, which speeds up large-scale path listings. Non-AWS S3 implementations may not support this API: consult the S3A documentation on how to revert to the v1 API.
+
+
+---
+
+* [HADOOP-14847](https://issues.apache.org/jira/browse/HADOOP-14847) | *Blocker* | **Remove Guava Supplier and change to java Supplier in AMRMClient and AMRMClientAysnc**
+
+AMRMClient#waitFor and AMRMClientAsync#waitFor now take java.util.function.Supplier as an argument, rather than com.google.common.base.Supplier.
+
+
+---
+
+* [HADOOP-14520](https://issues.apache.org/jira/browse/HADOOP-14520) | *Major* | **WASB: Block compaction for Azure Block Blobs**
+
+Block Compaction for Azure Block Blobs. When the number of blocks in a block blob is above 32000, the process of compaction replaces a sequence of small blocks with with one big block.
+
+
+---
+
+* [HDFS-12412](https://issues.apache.org/jira/browse/HDFS-12412) | *Major* | **Change ErasureCodingWorker.stripedReadPool to cached thread pool**
+
+Changed {{stripedReadPool}} to unbounded cachedThreadPool. User should combine {{dfs.datanode.ec.reconstruction.stripedblock.threads}} and {{dfs.namenode.replication.max-streams}} to tune recovery performance.
+
+
+---
+
+* [HDFS-12414](https://issues.apache.org/jira/browse/HDFS-12414) | *Major* | **Ensure to use CLI command to enable/disable erasure coding policy**
+
+dfs.namenode.ec.policies.enabled was removed in order to ensure there is only one approach to enable/disable erasure coding policies to avoid sync up.
+
+
+---
+
+* [HDFS-12438](https://issues.apache.org/jira/browse/HDFS-12438) | *Major* | **Rename dfs.datanode.ec.reconstruction.stripedblock.threads.size to dfs.datanode.ec.reconstruction.threads**
+
+<!-- markdown -->
+
+Config key `dfs.datanode.ec.reconstruction.stripedblock.threads.size` has been renamed to `dfs.datanode.ec.reconstruction.threads`.
+
+
+---
+
+* [HADOOP-14738](https://issues.apache.org/jira/browse/HADOOP-14738) | *Blocker* | **Remove S3N and obsolete bits of S3A; rework docs**
+
+\* The s3n:// client has been removed. Please upgrade to the s3a:// client.
+\* The s3a's original output stream has been removed, the "fast" output stream is the sole option available. There is no need to explicitly enable this, and trying to disable it (fs.s3a.fast.upload=false) will have no effect.
+
+
+---
+
+* [HDFS-7859](https://issues.apache.org/jira/browse/HDFS-7859) | *Major* | **Erasure Coding: Persist erasure coding policies in NameNode**
+
+Persist all built-in erasure coding policies and user defined erasure coding policies into NameNode fsImage and editlog reliably, so that all erasure coding policies remain consistent after NameNode restart.
+
+
+---
+
+* [HDFS-12395](https://issues.apache.org/jira/browse/HDFS-12395) | *Major* | **Support erasure coding policy operations in namenode edit log**
+
+See RN in HDFS-7859 as part of the work.
+
+
+---
+
+* [HADOOP-14670](https://issues.apache.org/jira/browse/HADOOP-14670) | *Major* | **Increase minimum cmake version for all platforms**
+
+CMake v3.1.0 is now the minimum version required to build Apache Hadoop's native components.
+
+
+---
+
+* [HDFS-11799](https://issues.apache.org/jira/browse/HDFS-11799) | *Major* | **Introduce a config to allow setting up write pipeline with fewer nodes than replication factor**
+
+Added new configuration "dfs.client.block.write.replace-datanode-on-failure.min-replication".
+
+ The minimum number of replications that are needed to not to fail
+ the write pipeline if new datanodes can not be found to replace
+ failed datanodes (could be due to network failure) in the write pipeline.
+ If the number of the remaining datanodes in the write pipeline is greater
+ than or equal to this property value, continue writing to the remaining nodes.
+ Otherwise throw exception.
+
+ If this is set to 0, an exception will be thrown, when a replacement
+ can not be found.
+
+
+---
+
+* [HDFS-12447](https://issues.apache.org/jira/browse/HDFS-12447) | *Major* | **Rename AddECPolicyResponse to AddErasureCodingPolicyResponse**
+
+HdfsAdmin#addErasureCodingPolicies now returns an AddErasureCodingPolicyResponse[] rather than AddECPolicyResponse[]. The corresponding RPC definition and PB message have also been renamed.
+
+
+---
+
+* [HDFS-7337](https://issues.apache.org/jira/browse/HDFS-7337) | *Critical* | **Configurable and pluggable erasure codec and policy**
+
+This allows users to:
+\* develop and plugin their own erasure codec and coders. The plugin will be loaded automatically from hadoop jars, the corresponding codec and coder will be registered for runtime use.
+\* define their own erasure coding policies thru an xml file and CLI command. The added policies will be persisted into fsimage.
+
+
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-6943. Update Yarn to YARN in
documentation. Contributed by Chetna Chaudhari.
Posted by jh...@apache.org.
YARN-6943. Update Yarn to YARN in documentation. Contributed by Chetna Chaudhari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9b525bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9b525bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9b525bd
Branch: refs/heads/YARN-5734
Commit: c9b525bd943fe7e098e72fb9be5623862c7ab5ff
Parents: 4a87773
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Oct 4 07:01:54 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Oct 4 07:01:54 2017 +0900
----------------------------------------------------------------------
.../hadoop/crypto/key/kms/server/TestKMS.java | 6 ++---
.../src/site/markdown/HadoopArchiveLogs.md | 6 ++---
.../src/site/markdown/SchedulerLoadSimulator.md | 20 ++++++++--------
hadoop-yarn-project/hadoop-yarn/README | 2 +-
.../yarn/api/records/ApplicationReport.java | 2 +-
.../records/ApplicationSubmissionContext.java | 2 +-
.../yarn/exceptions/YarnRuntimeException.java | 2 +-
.../hadoop/yarn/client/api/NMTokenCache.java | 2 +-
.../hadoop/yarn/client/api/YarnClient.java | 2 +-
.../yarn/YarnUncaughtExceptionHandler.java | 2 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 2 +-
.../hadoop/yarn/util/TrackingUriPlugin.java | 2 +-
.../hadoop/yarn/util/YarnVersionInfo.java | 12 +++++-----
.../registry/server/services/package-info.java | 2 +-
.../policies/FederationPolicyUtils.java | 4 ++--
.../yarn/server/MockResourceManagerFacade.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 6 ++---
.../linux/resources/CGroupsHandlerImpl.java | 6 ++---
.../linux/resources/TestCGroupsHandlerImpl.java | 6 ++---
.../scheduler/SchedulerNode.java | 2 +-
.../org/apache/hadoop/test/YarnTestDriver.java | 2 +-
.../clientrm/FederationClientInterceptor.java | 6 ++---
.../webapp/FederationInterceptorREST.java | 24 ++++++++++----------
.../MockDefaultRequestInterceptorREST.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 4 ++--
.../common/HBaseTimelineStorageUtils.java | 2 +-
.../server/webproxy/WebAppProxyServlet.java | 2 +-
.../hadoop-yarn/hadoop-yarn-ui/README.md | 6 ++---
.../src/main/webapp/app/index.html | 2 +-
.../src/main/webapp/config/default-config.js | 2 +-
30 files changed, 71 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index d6f2c25..45546f2 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -2208,7 +2208,7 @@ public class TestKMS {
"hadoop.kms.authentication.delegation-token.renew-interval.sec", "5");
writeConf(confDir, conf);
- // Running as a service (e.g. Yarn in practice).
+ // Running as a service (e.g. YARN in practice).
runServer(null, null, confDir, new KMSCallable<Void>() {
@Override
public Void call() throws Exception {
@@ -2223,7 +2223,7 @@ public class TestKMS {
final InetSocketAddress kmsAddr =
new InetSocketAddress(getKMSUrl().getHost(), getKMSUrl().getPort());
- // Job 1 (e.g. Yarn log aggregation job), with user DT.
+ // Job 1 (e.g. YARN log aggregation job), with user DT.
final Collection<Token<?>> job1Token = new HashSet<>();
doAs("client", new PrivilegedExceptionAction<Void>() {
@Override
@@ -2268,7 +2268,7 @@ public class TestKMS {
});
Assert.assertFalse(job1Token.isEmpty());
- // job 2 (e.g. Another Yarn log aggregation job, with user DT.
+ // job 2 (e.g. Another YARN log aggregation job, with user DT.
doAs("client", new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-tools/hadoop-archive-logs/src/site/markdown/HadoopArchiveLogs.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/site/markdown/HadoopArchiveLogs.md b/hadoop-tools/hadoop-archive-logs/src/site/markdown/HadoopArchiveLogs.md
index ce9ceba..3dc1349 100644
--- a/hadoop-tools/hadoop-archive-logs/src/site/markdown/HadoopArchiveLogs.md
+++ b/hadoop-tools/hadoop-archive-logs/src/site/markdown/HadoopArchiveLogs.md
@@ -21,7 +21,7 @@ Hadoop Archive Logs Guide
Overview
--------
-For clusters with a lot of Yarn aggregated logs, it can be helpful to combine
+For clusters with a lot of YARN aggregated logs, it can be helpful to combine
them into hadoop archives in order to reduce the number of small files, and
hence the stress on the NameNode. This tool provides an easy way to do this.
Aggregated logs in hadoop archives can still be read by the Job History Server
@@ -50,7 +50,7 @@ How to Archive Logs
to be eligible (default: 20)
-noProxy When specified, all processing will be
done as the user running this command (or
- the Yarn user if DefaultContainerExecutor
+ the YARN user if DefaultContainerExecutor
is in use). When not specified, all
processing will be done as the user who
owns that application; if the user
@@ -86,7 +86,7 @@ The tool works by performing the following procedure:
its aggregated log files with the resulting archive.
The ``-noProxy`` option makes the tool process everything as the user who is
-currently running it, or the Yarn user if DefaultContainerExecutor is in use.
+currently running it, or the YARN user if DefaultContainerExecutor is in use.
When not specified, all processing will be done by the user who owns that
application; if the user running this command is not allowed to impersonate that
user, it will fail. This is useful if you want an admin user to handle all
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
index d1848e8..d3f91f4 100644
--- a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -12,10 +12,10 @@
limitations under the License. See accompanying LICENSE file.
-->
-Yarn Scheduler Load Simulator (SLS)
+YARN Scheduler Load Simulator (SLS)
===================================
-* [Yarn Scheduler Load Simulator (SLS)](#Yarn_Scheduler_Load_Simulator_SLS)
+* [YARN Scheduler Load Simulator (SLS)](#Yarn_Scheduler_Load_Simulator_SLS)
* [Overview](#Overview)
* [Overview](#Overview)
* [Goals](#Goals)
@@ -39,11 +39,11 @@ Overview
### Overview
-The Yarn scheduler is a fertile area of interest with different implementations, e.g., Fifo, Capacity and Fair schedulers. Meanwhile, several optimizations are also made to improve scheduler performance for different scenarios and workload. Each scheduler algorithm has its own set of features, and drives scheduling decisions by many factors, such as fairness, capacity guarantee, resource availability, etc. It is very important to evaluate a scheduler algorithm very well before we deploy in a production cluster. Unfortunately, currently it is non-trivial to evaluate a scheduler algorithm. Evaluating in a real cluster is always time and cost consuming, and it is also very hard to find a large-enough cluster. Hence, a simulator which can predict how well a scheduler algorithm for some specific workload would be quite useful.
+The YARN scheduler is a fertile area of interest with different implementations, e.g., Fifo, Capacity and Fair schedulers. Meanwhile, several optimizations are also made to improve scheduler performance for different scenarios and workload. Each scheduler algorithm has its own set of features, and drives scheduling decisions by many factors, such as fairness, capacity guarantee, resource availability, etc. It is very important to evaluate a scheduler algorithm very well before we deploy in a production cluster. Unfortunately, currently it is non-trivial to evaluate a scheduler algorithm. Evaluating in a real cluster is always time and cost consuming, and it is also very hard to find a large-enough cluster. Hence, a simulator which can predict how well a scheduler algorithm for some specific workload would be quite useful.
-The Yarn Scheduler Load Simulator (SLS) is such a tool, which can simulate large-scale Yarn clusters and application loads in a single machine.This simulator would be invaluable in furthering Yarn by providing a tool for researchers and developers to prototype new scheduler features and predict their behavior and performance with reasonable amount of confidence, thereby aiding rapid innovation.
+The YARN Scheduler Load Simulator (SLS) is such a tool, which can simulate large-scale YARN clusters and application loads in a single machine.This simulator would be invaluable in furthering YARN by providing a tool for researchers and developers to prototype new scheduler features and predict their behavior and performance with reasonable amount of confidence, thereby aiding rapid innovation.
o
-The simulator will exercise the real Yarn `ResourceManager` removing the network factor by simulating `NodeManagers` and `ApplicationMasters` via handling and dispatching `NM`/`AMs` heartbeat events from within the same JVM. To keep tracking of scheduler behavior and performance, a scheduler wrapper will wrap the real scheduler.
+The simulator will exercise the real YARN `ResourceManager` removing the network factor by simulating `NodeManagers` and `ApplicationMasters` via handling and dispatching `NM`/`AMs` heartbeat events from within the same JVM. To keep tracking of scheduler behavior and performance, a scheduler wrapper will wrap the real scheduler.
The size of the cluster and the application load can be loaded from configuration files, which are generated from job history files directly by adopting [Apache Rumen](../hadoop-rumen/Rumen.html).
@@ -74,7 +74,7 @@ The following figure illustrates the implementation architecture of the simulato
-The simulator takes input of workload traces, or synthetic load distributions and generaters the cluster and applications information. For each NM and AM, the simulator builds a simulator to simulate their running. All NM/AM simulators run in a thread pool. The simulator reuses Yarn Resource Manager, and builds a wrapper out of the scheduler. The Scheduler Wrapper can track the scheduler behaviors and generates several logs, which are the outputs of the simulator and can be further analyzed.
+The simulator takes input of workload traces, or synthetic load distributions and generaters the cluster and applications information. For each NM and AM, the simulator builds a simulator to simulate their running. All NM/AM simulators run in a thread pool. The simulator reuses YARN Resource Manager, and builds a wrapper out of the scheduler. The Scheduler Wrapper can track the scheduler behaviors and generates several logs, which are the outputs of the simulator and can be further analyzed.
### Usecases
@@ -110,9 +110,9 @@ The following sections will describe how to use the simulator step by step. Befo
### Step 1: Configure Hadoop and the simulator
-Before we start, make sure Hadoop and the simulator are configured well. All configuration files for Hadoop and the simulator should be placed in directory `$HADOOP_ROOT/etc/hadoop`, where the `ResourceManager` and Yarn scheduler load their configurations. Directory `$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/` provides several example configurations, that can be used to start a demo.
+Before we start, make sure Hadoop and the simulator are configured well. All configuration files for Hadoop and the simulator should be placed in directory `$HADOOP_ROOT/etc/hadoop`, where the `ResourceManager` and YARN scheduler load their configurations. Directory `$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/` provides several example configurations, that can be used to start a demo.
-For configuration of Hadoop and Yarn scheduler, users can refer to Yarn’s website (<http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/>).
+For configuration of Hadoop and YARN scheduler, users can refer to Yarn’s website (<http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/>).
For the simulator, it loads configuration information from file `$HADOOP_ROOT/etc/hadoop/sls-runner.xml`.
@@ -244,7 +244,7 @@ The simulator supports two types of input files: the rumen traces and its own in
Metrics
-------
-The Yarn Scheduler Load Simulator has integrated [Metrics](http://metrics.codahale.com/) to measure the behaviors of critical components and operations, including running applications and containers, cluster available resources, scheduler operation timecost, et al. If the switch `yarn.sls.runner.metrics.switch` is set `ON`, `Metrics` will run and output it logs in `--output-dir` directory specified by users. Users can track these information during simulator running, and can also analyze these logs after running to evaluate the scheduler performance.
+The YARN Scheduler Load Simulator has integrated [Metrics](http://metrics.codahale.com/) to measure the behaviors of critical components and operations, including running applications and containers, cluster available resources, scheduler operation timecost, et al. If the switch `yarn.sls.runner.metrics.switch` is set `ON`, `Metrics` will run and output it logs in `--output-dir` directory specified by users. Users can track these information during simulator running, and can also analyze these logs after running to evaluate the scheduler performance.
### Real-time Tracking
@@ -320,7 +320,7 @@ Appendix
### Resources
-[YARN-1021](https://issues.apache.org/jira/browse/YARN-1021) is the main JIRA that introduces Yarn Scheduler Load Simulator to Hadoop Yarn project.
+[YARN-1021](https://issues.apache.org/jira/browse/YARN-1021) is the main JIRA that introduces YARN Scheduler Load Simulator to Hadoop YARN project.
[YARN-6363](https://issues.apache.org/jira/browse/YARN-6363) is the main JIRA that introduces the Synthetic Load Generator to SLS.
### SLS JSON input file format
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/README
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/README b/hadoop-yarn-project/hadoop-yarn/README
index 4e6aaa5..13835fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/README
+++ b/hadoop-yarn-project/hadoop-yarn/README
@@ -55,7 +55,7 @@ Modules
-------
YARN consists of multiple modules. The modules are listed below as per the directory structure:
-hadoop-yarn-api - Yarn's cross platform external interface
+hadoop-yarn-api - YARN's cross platform external interface
hadoop-yarn-common - Utilities which can be used by yarn clients and server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index 9e9ec3c..962bbba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -367,7 +367,7 @@ public abstract class ApplicationReport {
* Get the AMRM token of the application.
* <p>
* The AMRM token is required for AM to RM scheduling operations. For
- * managed Application Masters Yarn takes care of injecting it. For unmanaged
+ * managed Application Masters YARN takes care of injecting it. For unmanaged
* Applications Masters, the token must be obtained via this method and set
* in the {@link org.apache.hadoop.security.UserGroupInformation} of the
* current user.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index a6bbca7..38db60c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.yarn.util.Records;
* <li>
* maxAppAttempts. The maximum number of application attempts.
* It should be no larger than the global number of max attempts in the
- * Yarn configuration.
+ * YARN configuration.
* </li>
* <li>
* attemptFailuresValidityInterval. The default value is -1.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRuntimeException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRuntimeException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRuntimeException.java
index 6f8c6ed..e28dcec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRuntimeException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRuntimeException.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.exceptions;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-/** Base Yarn Exception.
+/** Base YARN Exception.
*
* NOTE: All derivatives of this exception, which may be thrown by a remote
* service, must include a String only constructor for the exception to be
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
index 0c349cc..c2c2620 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
@@ -35,7 +35,7 @@ import com.google.common.annotations.VisibleForTesting;
* NMTokenCache manages NMTokens required for an Application Master
* communicating with individual NodeManagers.
* <p>
- * By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use
+ * By default YARN client libraries {@link AMRMClient} and {@link NMClient} use
* {@link #getSingleton()} instance of the cache.
* <ul>
* <li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 60e7813..26c99e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -229,7 +229,7 @@ public abstract class YarnClient extends AbstractService {
* Get the AMRM token of the application.
* <p>
* The AMRM token is required for AM to RM scheduling operations. For
- * managed Application Masters Yarn takes care of injecting it. For unmanaged
+ * managed Application Masters YARN takes care of injecting it. For unmanaged
* Applications Masters, the token must be obtained via this method and set
* in the {@link org.apache.hadoop.security.UserGroupInformation} of the
* current user.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnUncaughtExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnUncaughtExceptionHandler.java
index 542fb22..7b4b774 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnUncaughtExceptionHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/YarnUncaughtExceptionHandler.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.util.ShutdownHookManager;
* This class is intended to be installed by calling
* {@link Thread#setDefaultUncaughtExceptionHandler(UncaughtExceptionHandler)}
* In the main entry point. It is intended to try and cleanly shut down
- * programs using the Yarn Event framework.
+ * programs using the YARN Event framework.
*
* Note: Right now it only will shut down the program if a Error is caught, but
* not any other exception. Anything else is just logged.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 9235e7d..27191be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
/**
- * Yarn internal application-related utilities
+ * YARN internal application-related utilities
*/
@Private
public class Apps {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
index c3def07..d29e52f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TrackingUriPlugin.java
@@ -27,7 +27,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.yarn.api.records.ApplicationId;
/**
- * Plugin to derive a tracking URL from a Yarn Application ID
+ * Plugin to derive a tracking URL from a YARN Application ID
*
*/
@InterfaceAudience.LimitedPrivate({"MapReduce"})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
index 9daee33..e515321 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/YarnVersionInfo.java
@@ -38,8 +38,8 @@ public class YarnVersionInfo extends VersionInfo {
super("yarn");
}
/**
- * Get the Yarn version.
- * @return the Yarn version string, eg. "0.6.3-dev"
+ * Get the YARN version.
+ * @return the YARN version string, eg. "0.6.3-dev"
*/
public static String getVersion() {
return YARN_VERSION_INFO._getVersion();
@@ -62,7 +62,7 @@ public class YarnVersionInfo extends VersionInfo {
}
/**
- * The date that Yarn was compiled.
+ * The date that YARN was compiled.
* @return the compilation date in unix date format
*/
public static String getDate() {
@@ -78,14 +78,14 @@ public class YarnVersionInfo extends VersionInfo {
}
/**
- * Get the subversion URL for the root Yarn directory.
+ * Get the subversion URL for the root YARN directory.
*/
public static String getUrl() {
return YARN_VERSION_INFO._getUrl();
}
/**
- * Get the checksum of the source files from which Yarn was
+ * Get the checksum of the source files from which YARN was
* built.
**/
public static String getSrcChecksum() {
@@ -102,7 +102,7 @@ public class YarnVersionInfo extends VersionInfo {
public static void main(String[] args) {
LOG.debug("version: "+ getVersion());
- System.out.println("Yarn " + getVersion());
+ System.out.println("YARN " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
System.out.println("From source with checksum " + getSrcChecksum());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/package-info.java
index fe2a0a8..4bca195 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/services/package-info.java
@@ -21,7 +21,7 @@
* <ul>
* <li>
* The {@link org.apache.hadoop.registry.server.services.RegistryAdminService}
- * extends the shared Yarn Registry client with registry setup and
+ * extends the shared YARN Registry client with registry setup and
* (potentially asynchronous) administrative actions.
* </li>
* <li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
index 97e4848..7716a6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyUtils.java
@@ -80,7 +80,7 @@ public final class FederationPolicyUtils {
* and configuration as fallback.
*
* @param queue the queue of the application
- * @param conf the Yarn configuration
+ * @param conf the YARN configuration
* @param federationFacade state store facade
* @return SubClusterPolicyConfiguration recreated
*/
@@ -139,7 +139,7 @@ public final class FederationPolicyUtils {
*
* @param queue the queue of the application
* @param oldPolicy the previous policy instance (can be null)
- * @param conf the Yarn configuration
+ * @param conf the YARN configuration
* @param federationFacade state store facade
* @param homeSubClusterId home sub-cluster id
* @return FederationAMRMProxyPolicy recreated
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 769296b..628c781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -187,7 +187,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
final private AtomicInteger applicationCounter = new AtomicInteger(0);
// True if the Mock RM is running, false otherwise.
- // This property allows us to write tests for specific scenario as Yarn RM
+ // This property allows us to write tests for specific scenario as YARN RM
// down e.g. network issue, failover.
private boolean isRunning;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index d12892e..38eb636 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -1716,7 +1716,7 @@ public class ContainerManagerImpl extends CompositeService implements
* @param containerId Container Id.
* @param autoCommit Auto Commit flag.
* @param reInitLaunchContext Target Launch Context.
- * @throws YarnException Yarn Exception.
+ * @throws YarnException YARN Exception.
*/
public void reInitializeContainer(ContainerId containerId,
ContainerLaunchContext reInitLaunchContext, boolean autoCommit)
@@ -1743,7 +1743,7 @@ public class ContainerManagerImpl extends CompositeService implements
* Rollback the last reInitialization, if possible.
* @param containerId Container ID.
* @return Rollback Response.
- * @throws YarnException Yarn Exception.
+ * @throws YarnException YARN Exception.
*/
@Override
public RollbackResponse rollbackLastReInitialization(ContainerId containerId)
@@ -1764,7 +1764,7 @@ public class ContainerManagerImpl extends CompositeService implements
* Commit last reInitialization after which no rollback will be possible.
* @param containerId Container ID.
* @return Commit Response.
- * @throws YarnException Yarn Exception.
+ * @throws YarnException YARN Exception.
*/
@Override
public CommitResponse commitLastReInitialization(ContainerId containerId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 1c6385d..619a65b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -370,7 +370,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
}
// We are working with a pre-mounted contoller
- // Make sure that Yarn cgroup hierarchy path exists
+ // Make sure that YARN cgroup hierarchy path exists
initializePreMountedCGroupController(controller);
}
@@ -378,9 +378,9 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* This function is called when the administrator opted
* to use a pre-mounted cgroup controller.
* There are two options.
- * 1. Yarn hierarchy already exists. We verify, whether we have write access
+ * 1. YARN hierarchy already exists. We verify, whether we have write access
* in this case.
- * 2. Yarn hierarchy does not exist, yet. We create it in this case.
+ * 2. YARN hierarchy does not exist, yet. We create it in this case.
* @param controller the controller being initialized
* @throws ResourceHandlerException yarn hierarchy cannot be created or
* accessed for any reason
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index 996fff0..0d7c097 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -118,7 +118,7 @@ public class TestCGroupsHandlerImpl {
/**
* Create configuration where the cgroups are premounted.
- * @param myHierarchy Yarn cgroup
+ * @param myHierarchy YARN cgroup
* @return configuration object
*/
private Configuration createNoMountConfiguration(String myHierarchy) {
@@ -396,7 +396,7 @@ public class TestCGroupsHandlerImpl {
File mtab = createPremountedCgroups(parentDir, false);
File mountPoint = new File(parentDir, "cpu");
- // Initialize Yarn classes
+ // Initialize YARN classes
Configuration confNoMount = createNoMountConfiguration(myHierarchy);
CGroupsHandlerImpl cGroupsHandler = new CGroupsHandlerImpl(confNoMount,
privilegedOperationExecutorMock, mtab.getAbsolutePath());
@@ -555,7 +555,7 @@ public class TestCGroupsHandlerImpl {
assertTrue("Could not create dirs",
new File(newMountPoint, "cpu").mkdirs());
- // Initialize Yarn classes
+ // Initialize YARN classes
Configuration confMount = createMountConfiguration();
confMount.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH,
parentDir.getAbsolutePath() + Path.SEPARATOR + newMountPointDir);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 90fa3e4..05dbf1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -129,7 +129,7 @@ public abstract class SchedulerNode {
* Typically this is the 'hostname' reported by the node, but it could be
* configured to be 'hostname:port' reported by the node via the
* {@link YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} constant.
- * The main usecase of this is Yarn minicluster to be able to differentiate
+ * The main usecase of this is YARN minicluster to be able to differentiate
* node manager instances by their port number.
* @return Name of the node for scheduling matching decisions.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/test/YarnTestDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/test/YarnTestDriver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/test/YarnTestDriver.java
index 8874ed8..a28d46e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/test/YarnTestDriver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/test/YarnTestDriver.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.util.ProgramDriver;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.TestZKRMStateStorePerf;
/**
- * Driver for Yarn tests.
+ * Driver for YARN tests.
*
*/
public class YarnTestDriver {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index b8f8a9f..07eaf97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -212,7 +212,7 @@ public class FederationClientInterceptor
}
/**
- * Yarn Router forwards every getNewApplication requests to any RM. During
+ * YARN Router forwards every getNewApplication requests to any RM. During
* this operation there will be no communication with the State Store. The
* Router will forward the requests to any SubCluster. The Router will retry
* to submit the request on #numSubmitRetries different SubClusters. The
@@ -431,7 +431,7 @@ public class FederationClientInterceptor
}
/**
- * The Yarn Router will forward to the respective Yarn RM in which the AM is
+ * The YARN Router will forward to the respective YARN RM in which the AM is
* running.
*
* Possible failures and behaviors:
@@ -496,7 +496,7 @@ public class FederationClientInterceptor
}
/**
- * The Yarn Router will forward to the respective Yarn RM in which the AM is
+ * The YARN Router will forward to the respective YARN RM in which the AM is
* running.
*
* Possible failure:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
index bfd35c5..5adcc62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
@@ -228,7 +228,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
/**
- * Yarn Router forwards every getNewApplication requests to any RM. During
+ * YARN Router forwards every getNewApplication requests to any RM. During
* this operation there will be no communication with the State Store. The
* Router will forward the requests to any SubCluster. The Router will retry
* to submit the request on #numSubmitRetries different SubClusters. The
@@ -497,7 +497,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
/**
- * The Yarn Router will forward to the respective Yarn RM in which the AM is
+ * The YARN Router will forward to the respective YARN RM in which the AM is
* running.
* <p>
* Possible failure:
@@ -552,7 +552,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
/**
- * The Yarn Router will forward to the respective Yarn RM in which the AM is
+ * The YARN Router will forward to the respective YARN RM in which the AM is
* running.
* <p>
* Possible failures and behaviors:
@@ -606,7 +606,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
/**
- * The Yarn Router will forward the request to all the Yarn RMs in parallel,
+ * The YARN Router will forward the request to all the YARN RMs in parallel,
* after that it will group all the ApplicationReports by the ApplicationId.
* <p>
* Possible failure:
@@ -615,8 +615,8 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
* <p>
* Router: the Client will timeout and resubmit the request.
* <p>
- * ResourceManager: the Router calls each Yarn RM in parallel by using one
- * thread for each Yarn RM. In case a Yarn RM fails, a single call will
+ * ResourceManager: the Router calls each YARN RM in parallel by using one
+ * thread for each YARN RM. In case a YARN RM fails, a single call will
* timeout. However the Router will merge the ApplicationReports it got, and
* provides a partial list to the client.
* <p>
@@ -692,14 +692,14 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
return null;
}
- // Merge all the application reports got from all the available Yarn RMs
+ // Merge all the application reports got from all the available YARN RMs
return RouterWebServiceUtil.mergeAppsInfo(apps.getApps(),
returnPartialReport);
}
/**
- * The Yarn Router will forward to the request to all the SubClusters to find
+ * The YARN Router will forward to the request to all the SubClusters to find
* where the node is running.
* <p>
* Possible failure:
@@ -779,7 +779,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
/**
- * The Yarn Router will forward the request to all the Yarn RMs in parallel,
+ * The YARN Router will forward the request to all the YARN RMs in parallel,
* after that it will remove all the duplicated NodeInfo by using the NodeId.
* <p>
* Possible failure:
@@ -788,8 +788,8 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
* <p>
* Router: the Client will timeout and resubmit the request.
* <p>
- * ResourceManager: the Router calls each Yarn RM in parallel by using one
- * thread for each Yarn RM. In case a Yarn RM fails, a single call will
+ * ResourceManager: the Router calls each YARN RM in parallel by using one
+ * thread for each YARN RM. In case a YARN RM fails, a single call will
* timeout. However the Router will use the NodesInfo it got, and provides a
* partial list to the client.
* <p>
@@ -850,7 +850,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
// Delete duplicate from all the node reports got from all the available
- // Yarn RMs. Nodes can be moved from one subclusters to another. In this
+ // YARN RMs. Nodes can be moved from one subclusters to another. In this
// operation they result LOST/RUNNING in the previous SubCluster and
// NEW/RUNNING in the new one.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
index 6afecae..9f54582 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java
@@ -57,7 +57,7 @@ public class MockDefaultRequestInterceptorREST
LoggerFactory.getLogger(MockDefaultRequestInterceptorREST.class);
final private AtomicInteger applicationCounter = new AtomicInteger(0);
// True if the Mock RM is running, false otherwise.
- // This property allows us to write tests for specific scenario as Yarn RM
+ // This property allows us to write tests for specific scenario as YARN RM
// down e.g. network issue, failover.
private boolean isRunning = true;
private HashSet<ApplicationId> applicationMap = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 3b1247a..27aa916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -105,11 +105,11 @@ import static org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourc
/**
* <p>
- * Embedded Yarn minicluster for testcases that need to interact with a cluster.
+ * Embedded YARN minicluster for testcases that need to interact with a cluster.
* </p>
* <p>
* In a real cluster, resource request matching is done using the hostname, and
- * by default Yarn minicluster works in the exact same way as a real cluster.
+ * by default YARN minicluster works in the exact same way as a real cluster.
* </p>
* <p>
* If a testcase needs to use multiple nodes and exercise resource request
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
index b0d8527..c115b18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
@@ -263,7 +263,7 @@ public final class HBaseTimelineStorageUtils {
}
/**
- * @param conf Yarn configuration. Used to see if there is an explicit config
+ * @param conf YARN configuration. Used to see if there is an explicit config
* pointing to the HBase config file to read. It should not be null
* or a NullPointerException will be thrown.
* @return a configuration with the HBase configuration from the classpath,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index e1588c1..f21ff2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -613,7 +613,7 @@ public class WebAppProxyServlet extends HttpServlet {
* <p>
* <I>Do not remove</I>
* <p>
- * Yarn isn't currently serializing this class, but findbugs
+ * YARN isn't currently serializing this class, but findbugs
* complains in its absence.
*
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/README.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/README.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/README.md
index f67f351..e4135ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/README.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/README.md
@@ -15,9 +15,9 @@
limitations under the License.
-->
-# Yarn UI
+# YARN UI
-The Yarn UI is an Ember based web-app that provides visualization of the applications running on the Apache Hadoop YARN framework.
+The YARN UI is an Ember based web-app that provides visualization of the applications running on the Apache Hadoop YARN framework.
## Configurations
@@ -49,7 +49,7 @@ You will need the following things properly installed on your computer.
**Warning: Do not edit the _package.json_ or _bower.json_ files manually. This could make them out-of-sync with the respective lock or shrinkwrap files.**
-Yarn UI has replaced NPM with Yarn package manager. And hence Yarn would be used to manage dependencies defined in package.json.
+YARN UI has replaced NPM with Yarn package manager. And hence Yarn would be used to manage dependencies defined in package.json.
* Please use the Yarn and Bower command-line tools to add new dependencies. And the tool version must be same as those defined in Prerequisites section.
* Once any dependency is added:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/index.html
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/index.html b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/index.html
index f727454..9fcf0ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/index.html
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/index.html
@@ -21,7 +21,7 @@
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
- <title>YarnUi</title>
+ <title>YARN</title>
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9b525bd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
index 8ab7ce1..acde40c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/config/default-config.js
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-module.exports = { // Yarn UI App configurations
+module.exports = { // YARN UI App configurations
hosts: {
localBaseAddress: "",
timelineWebAddress: "localhost:8188",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-14928. Update site release
notes for 3.0.0 GA.
Posted by jh...@apache.org.
HADOOP-14928. Update site release notes for 3.0.0 GA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cae1c734
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cae1c734
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cae1c734
Branch: refs/heads/YARN-5734
Commit: cae1c734febc67fdecb754e386bf7df4241832db
Parents: 2df1b2a
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Oct 4 14:59:40 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Oct 4 14:59:40 2017 -0700
----------------------------------------------------------------------
hadoop-project/src/site/markdown/index.md.vm | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cae1c734/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
index 1526f59..d9e645b 100644
--- a/hadoop-project/src/site/markdown/index.md.vm
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -18,9 +18,8 @@ Apache Hadoop ${project.version}
Apache Hadoop ${project.version} incorporates a number of significant
enhancements over the previous major release line (hadoop-2.x).
-This is an alpha release to facilitate testing and the collection of
-feedback from downstream application developers and users. There are
-no guarantees regarding API stability or quality.
+This release is generally available (GA), meaning that it represents a point of
+API stability and quality that we consider production-ready.
Overview
========
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-5949. Add pluggable
configuration ACL policy interface and implementation. (Jonathan Hung via
wangda)
Posted by jh...@apache.org.
YARN-5949. Add pluggable configuration ACL policy interface and implementation. (Jonathan Hung via wangda)
Change-Id: Ib98e82ff753bede21fcab2e6ca9ec1e7a5a2008f
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f3fce9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f3fce9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f3fce9
Branch: refs/heads/YARN-5734
Commit: 87f3fce938dd2b177f1fe5148299987f833f43b7
Parents: 4006c67
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 22 13:38:31 2017 -0700
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:53 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../src/main/resources/yarn-default.xml | 11 ++
.../ConfigurationMutationACLPolicy.java | 47 ++++++
.../ConfigurationMutationACLPolicyFactory.java | 49 ++++++
.../DefaultConfigurationMutationACLPolicy.java | 45 ++++++
.../scheduler/MutableConfScheduler.java | 19 ++-
.../scheduler/MutableConfigurationProvider.java | 8 +-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../conf/MutableCSConfigurationProvider.java | 151 +++++++++++++++++-
...ueueAdminConfigurationMutationACLPolicy.java | 96 ++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 131 +---------------
.../TestConfigurationMutationACLPolicies.java | 154 +++++++++++++++++++
.../TestMutableCSConfigurationProvider.java | 40 +++--
13 files changed, 610 insertions(+), 150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 92b9697..b603135 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -680,6 +680,9 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_CONFIGURATION_STORE =
MEMORY_CONFIGURATION_STORE;
+ public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
+ YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
+
public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 73f4717..e124c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3379,4 +3379,15 @@
<value>memory</value>
</property>
+ <property>
+ <description>
+ The class to use for configuration mutation ACL policy if using a mutable
+ configuration provider. Controls whether a mutation request is allowed.
+ The DefaultConfigurationMutationACLPolicy checks if the requestor is a
+ YARN admin.
+ </description>
+ <name>yarn.scheduler.configuration.mutation.acl-policy.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..724487b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicy.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Interface for determining whether configuration mutations are allowed.
+ */
+public interface ConfigurationMutationACLPolicy {
+
+ /**
+ * Initialize ACL policy with configuration and RMContext.
+ * @param conf Configuration to initialize with.
+ * @param rmContext rmContext
+ */
+ void init(Configuration conf, RMContext rmContext);
+
+ /**
+ * Check if mutation is allowed.
+ * @param user User issuing the request
+ * @param confUpdate configurations to be updated
+ * @return whether provided mutation is allowed or not
+ */
+ boolean isMutationAllowed(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
new file mode 100644
index 0000000..2898785
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ConfigurationMutationACLPolicyFactory.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Factory class for creating instances of
+ * {@link ConfigurationMutationACLPolicy}.
+ */
+public final class ConfigurationMutationACLPolicyFactory {
+
+ private static final Log LOG = LogFactory.getLog(
+ ConfigurationMutationACLPolicyFactory.class);
+
+ private ConfigurationMutationACLPolicyFactory() {
+ // Unused.
+ }
+
+ public static ConfigurationMutationACLPolicy getPolicy(Configuration conf) {
+ Class<? extends ConfigurationMutationACLPolicy> policyClass =
+ conf.getClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ LOG.info("Using ConfigurationMutationACLPolicy implementation - " +
+ policyClass);
+ return ReflectionUtils.newInstance(policyClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..680c3b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/DefaultConfigurationMutationACLPolicy.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+/**
+ * Default configuration mutation ACL policy. Checks if user is YARN admin.
+ */
+public class DefaultConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private YarnAuthorizationProvider authorizer;
+
+ @Override
+ public void init(Configuration conf, RMContext rmContext) {
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ return authorizer.isAdmin(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
index 35e36e1..93a935e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java
@@ -17,10 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
-import java.util.Map;
/**
* Interface for a scheduler that supports changing configuration at runtime.
@@ -31,10 +32,22 @@ public interface MutableConfScheduler extends ResourceScheduler {
/**
* Update the scheduler's configuration.
* @param user Caller of this update
- * @param confUpdate key-value map of the configuration update
+ * @param confUpdate configuration update
* @throws IOException if update is invalid
*/
void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException;
+ QueueConfigsUpdateInfo confUpdate) throws IOException;
+ /**
+ * Get the scheduler configuration.
+ * @return the scheduler configuration
+ */
+ Configuration getConfiguration();
+
+ /**
+ * Get queue object based on queue name.
+ * @param queueName the queue name
+ * @return the queue object
+ */
+ Queue getQueue(String queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 889c3bc..f04c128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
import java.io.IOException;
-import java.util.Map;
/**
* Interface for allowing changing scheduler configurations.
@@ -32,7 +34,7 @@ public interface MutableConfigurationProvider {
* @param confUpdate Key-value pairs for configurations to be updated.
* @throws IOException if scheduler could not be reinitialized
*/
- void mutateConfiguration(String user, Map<String, String> confUpdate)
- throws IOException;
+ void mutateConfiguration(UserGroupInformation user, QueueConfigsUpdateInfo
+ confUpdate) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 7c5839b..4f89f7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -137,6 +137,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Placeme
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -649,6 +650,7 @@ public class CapacityScheduler extends
preemptionManager.refreshQueues(null, this.getRootQueue());
}
+ @Override
public CSQueue getQueue(String queueName) {
if (queueName == null) {
return null;
@@ -2615,10 +2617,10 @@ public class CapacityScheduler extends
@Override
public void updateConfiguration(UserGroupInformation user,
- Map<String, String> confUpdate) throws IOException {
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
if (csConfProvider instanceof MutableConfigurationProvider) {
((MutableConfigurationProvider) csConfProvider).mutateConfiguration(
- user.getShortUserName(), confUpdate);
+ user, confUpdate);
} else {
throw new UnsupportedOperationException("Configured CS configuration " +
"provider does not support updating configuration.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ea1b3c0..8b879b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -18,14 +18,27 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import com.google.common.base.Joiner;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
@@ -38,6 +51,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
private Configuration schedConf;
private YarnConfigurationStore confStore;
+ private ConfigurationMutationACLPolicy aclMutationPolicy;
private RMContext rmContext;
private Configuration conf;
@@ -68,6 +82,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
schedConf.set(kv.getKey(), kv.getValue());
}
confStore.initialize(config, schedConf);
+ this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory
+ .getPolicy(config);
+ aclMutationPolicy.init(config, rmContext);
this.conf = config;
}
@@ -80,12 +97,17 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public void mutateConfiguration(String user,
- Map<String, String> confUpdate) throws IOException {
+ public void mutateConfiguration(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) throws IOException {
+ if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) {
+ throw new AccessControlException("User is not admin of all modified" +
+ " queues.");
+ }
Configuration oldConf = new Configuration(schedConf);
- LogMutation log = new LogMutation(confUpdate, user);
+ Map<String, String> kvUpdate = constructKeyValueConfUpdate(confUpdate);
+ LogMutation log = new LogMutation(kvUpdate, user.getShortUserName());
long id = confStore.logMutation(log);
- for (Map.Entry<String, String> kv : confUpdate.entrySet()) {
+ for (Map.Entry<String, String> kv : kvUpdate.entrySet()) {
if (kv.getValue() == null) {
schedConf.unset(kv.getKey());
} else {
@@ -101,4 +123,125 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
confStore.confirmMutation(id, true);
}
+
+
+ private Map<String, String> constructKeyValueConfUpdate(
+ QueueConfigsUpdateInfo mutationInfo) throws IOException {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ CapacitySchedulerConfiguration proposedConf =
+ new CapacitySchedulerConfiguration(cs.getConfiguration(), false);
+ Map<String, String> confUpdate = new HashMap<>();
+ for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
+ removeQueue(queueToRemove, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
+ addQueue(addQueueInfo, proposedConf, confUpdate);
+ }
+ for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
+ updateQueue(updateQueueInfo, proposedConf, confUpdate);
+ }
+ return confUpdate;
+ }
+
+ private void removeQueue(
+ String queueToRemove, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (queueToRemove == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queueName = queueToRemove.substring(
+ queueToRemove.lastIndexOf('.') + 1);
+ CSQueue queue = cs.getQueue(queueName);
+ if (queue == null ||
+ !queue.getQueuePath().equals(queueToRemove)) {
+ throw new IOException("Queue " + queueToRemove + " not found");
+ } else if (queueToRemove.lastIndexOf('.') == -1) {
+ throw new IOException("Can't remove queue " + queueToRemove);
+ }
+ String parentQueuePath = queueToRemove.substring(0, queueToRemove
+ .lastIndexOf('.'));
+ String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
+ List<String> newSiblingQueues = new ArrayList<>();
+ for (String siblingQueue : siblingQueues) {
+ if (!siblingQueue.equals(queueName)) {
+ newSiblingQueues.add(siblingQueue);
+ }
+ }
+ proposedConf.setQueues(parentQueuePath, newSiblingQueues
+ .toArray(new String[0]));
+ String queuesConfig = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES;
+ if (newSiblingQueues.size() == 0) {
+ confUpdate.put(queuesConfig, null);
+ } else {
+ confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
+ }
+ for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
+ ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
+ .entrySet()) {
+ proposedConf.unset(confRemove.getKey());
+ confUpdate.put(confRemove.getKey(), null);
+ }
+ }
+ }
+
+ private void addQueue(
+ QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) throws IOException {
+ if (addInfo == null) {
+ return;
+ } else {
+ CapacityScheduler cs = (CapacityScheduler) rmContext.getScheduler();
+ String queuePath = addInfo.getQueue();
+ String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
+ if (cs.getQueue(queueName) != null) {
+ throw new IOException("Can't add existing queue " + queuePath);
+ } else if (queuePath.lastIndexOf('.') == -1) {
+ throw new IOException("Can't add invalid queue " + queuePath);
+ }
+ String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
+ String[] siblings = proposedConf.getQueues(parentQueue);
+ List<String> siblingQueues = siblings == null ? new ArrayList<>() :
+ new ArrayList<>(Arrays.<String>asList(siblings));
+ siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
+ proposedConf.setQueues(parentQueue,
+ siblingQueues.toArray(new String[0]));
+ confUpdate.put(CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT
+ + CapacitySchedulerConfiguration.QUEUES,
+ Joiner.on(',').join(siblingQueues));
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
+
+ private void updateQueue(QueueConfigInfo updateInfo,
+ CapacitySchedulerConfiguration proposedConf,
+ Map<String, String> confUpdate) {
+ if (updateInfo == null) {
+ return;
+ } else {
+ String queuePath = updateInfo.getQueue();
+ String keyPrefix = CapacitySchedulerConfiguration.PREFIX
+ + queuePath + CapacitySchedulerConfiguration.DOT;
+ for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
+ if (kv.getValue() == null) {
+ proposedConf.unset(keyPrefix + kv.getKey());
+ } else {
+ proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
new file mode 100644
index 0000000..1f94c1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueAdminConfigurationMutationACLPolicy.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A configuration mutation ACL policy which checks that user has admin
+ * privileges on all queues they are changing.
+ */
+public class QueueAdminConfigurationMutationACLPolicy implements
+ ConfigurationMutationACLPolicy {
+
+ private RMContext rmContext;
+
+ @Override
+ public void init(Configuration conf, RMContext context) {
+ this.rmContext = context;
+ }
+
+ @Override
+ public boolean isMutationAllowed(UserGroupInformation user,
+ QueueConfigsUpdateInfo confUpdate) {
+ Set<String> queues = new HashSet<>();
+ for (QueueConfigInfo addQueueInfo : confUpdate.getAddQueueInfo()) {
+ queues.add(addQueueInfo.getQueue());
+ }
+ for (String removeQueue : confUpdate.getRemoveQueueInfo()) {
+ queues.add(removeQueue);
+ }
+ for (QueueConfigInfo updateQueueInfo : confUpdate.getUpdateQueueInfo()) {
+ queues.add(updateQueueInfo.getQueue());
+ }
+ for (String queuePath : queues) {
+ String queueName = queuePath.lastIndexOf('.') != -1 ?
+ queuePath.substring(queuePath.lastIndexOf('.') + 1) : queuePath;
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ String parentPath = queuePath;
+ // TODO: handle global config change.
+ while (queueInfo == null) {
+ // We are adding a queue (whose parent we are possibly also adding).
+ // Check ACL of lowest parent queue which already exists.
+ parentPath = parentPath.substring(0, parentPath.lastIndexOf('.'));
+ String parentName = parentPath.lastIndexOf('.') != -1 ?
+ parentPath.substring(parentPath.lastIndexOf('.') + 1) : parentPath;
+ try {
+ queueInfo = rmContext.getScheduler()
+ .getQueueInfo(parentName, false, false);
+ } catch (IOException e) {
+ // Queue is not found, do nothing.
+ }
+ }
+ Queue queue = ((MutableConfScheduler) rmContext.getScheduler())
+ .getQueue(queueInfo.getQueueName());
+ if (queue != null && !queue.hasAccess(QueueACL.ADMINISTER_QUEUE, user)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 215e511..d95465b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -142,7 +142,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -2484,10 +2483,8 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
callerUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException, YarnException {
- Map<String, String> confUpdate =
- constructKeyValueConfUpdate(mutationInfo);
- ((CapacityScheduler) scheduler).updateConfiguration(callerUGI,
- confUpdate);
+ ((MutableConfScheduler) scheduler).updateConfiguration(callerUGI,
+ mutationInfo);
return null;
}
});
@@ -2499,129 +2496,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
"successfully applied.").build();
} else {
return Response.status(Status.BAD_REQUEST)
- .entity("Configuration change only supported by CapacityScheduler.")
+ .entity("Configuration change only supported by " +
+ "MutableConfScheduler.")
.build();
}
}
-
- private Map<String, String> constructKeyValueConfUpdate(
- QueueConfigsUpdateInfo mutationInfo) throws IOException {
- CapacitySchedulerConfiguration currentConf =
- ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
- CapacitySchedulerConfiguration proposedConf =
- new CapacitySchedulerConfiguration(currentConf, false);
- Map<String, String> confUpdate = new HashMap<>();
- for (String queueToRemove : mutationInfo.getRemoveQueueInfo()) {
- removeQueue(queueToRemove, proposedConf, confUpdate);
- }
- for (QueueConfigInfo addQueueInfo : mutationInfo.getAddQueueInfo()) {
- addQueue(addQueueInfo, proposedConf, confUpdate);
- }
- for (QueueConfigInfo updateQueueInfo : mutationInfo.getUpdateQueueInfo()) {
- updateQueue(updateQueueInfo, proposedConf, confUpdate);
- }
- return confUpdate;
- }
-
- private void removeQueue(
- String queueToRemove, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (queueToRemove == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queueName = queueToRemove.substring(
- queueToRemove.lastIndexOf('.') + 1);
- CSQueue queue = cs.getQueue(queueName);
- if (queue == null ||
- !queue.getQueuePath().equals(queueToRemove)) {
- throw new IOException("Queue " + queueToRemove + " not found");
- } else if (queueToRemove.lastIndexOf('.') == -1) {
- throw new IOException("Can't remove queue " + queueToRemove);
- }
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- String[] siblingQueues = proposedConf.getQueues(parentQueuePath);
- List<String> newSiblingQueues = new ArrayList<>();
- for (String siblingQueue : siblingQueues) {
- if (!siblingQueue.equals(queueName)) {
- newSiblingQueues.add(siblingQueue);
- }
- }
- proposedConf.setQueues(parentQueuePath, newSiblingQueues
- .toArray(new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX +
- parentQueuePath + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES;
- if (newSiblingQueues.size() == 0) {
- confUpdate.put(queuesConfig, null);
- } else {
- confUpdate.put(queuesConfig, Joiner.on(',').join(newSiblingQueues));
- }
- for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove.replaceAll("\\.", "\\.") + "\\..*")
- .entrySet()) {
- proposedConf.unset(confRemove.getKey());
- confUpdate.put(confRemove.getKey(), null);
- }
- }
- }
-
- private void addQueue(
- QueueConfigInfo addInfo, CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) throws IOException {
- if (addInfo == null) {
- return;
- } else {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String queuePath = addInfo.getQueue();
- String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (cs.getQueue(queueName) != null) {
- throw new IOException("Can't add existing queue " + queuePath);
- } else if (queuePath.lastIndexOf('.') == -1) {
- throw new IOException("Can't add invalid queue " + queuePath);
- }
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String[] siblings = proposedConf.getQueues(parentQueue);
- List<String> siblingQueues = siblings == null ? new ArrayList<>() :
- new ArrayList<>(Arrays.<String>asList(siblings));
- siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
- proposedConf.setQueues(parentQueue,
- siblingQueues.toArray(new String[0]));
- confUpdate.put(CapacitySchedulerConfiguration.PREFIX +
- parentQueue + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES,
- Joiner.on(',').join(siblingQueues));
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
-
- private void updateQueue(QueueConfigInfo updateInfo,
- CapacitySchedulerConfiguration proposedConf,
- Map<String, String> confUpdate) {
- if (updateInfo == null) {
- return;
- } else {
- String queuePath = updateInfo.getQueue();
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX +
- queuePath + CapacitySchedulerConfiguration.DOT;
- for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
- if (kv.getValue() == null) {
- proposedConf.unset(keyPrefix + kv.getKey());
- } else {
- proposedConf.set(keyPrefix + kv.getKey(), kv.getValue());
- }
- confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
new file mode 100644
index 0000000..4016dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestConfigurationMutationACLPolicies.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestConfigurationMutationACLPolicies {
+
+ private ConfigurationMutationACLPolicy policy;
+ private RMContext rmContext;
+ private MutableConfScheduler scheduler;
+
+ private static final UserGroupInformation GOOD_USER = UserGroupInformation
+ .createUserForTesting("goodUser", new String[] {});
+ private static final UserGroupInformation BAD_USER = UserGroupInformation
+ .createUserForTesting("badUser", new String[] {});
+ private static final Map<String, String> EMPTY_MAP =
+ Collections.<String, String>emptyMap();
+
+ @Before
+ public void setUp() throws IOException {
+ rmContext = mock(RMContext.class);
+ scheduler = mock(MutableConfScheduler.class);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ mockQueue("a", scheduler);
+ mockQueue("b", scheduler);
+ mockQueue("b1", scheduler);
+ }
+
+ private void mockQueue(String queueName, MutableConfScheduler scheduler)
+ throws IOException {
+ QueueInfo queueInfo = QueueInfo.newInstance(queueName, 0, 0, 0, null, null,
+ null, null, null, null, false);
+ when(scheduler.getQueueInfo(eq(queueName), anyBoolean(), anyBoolean()))
+ .thenReturn(queueInfo);
+ Queue queue = mock(Queue.class);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(GOOD_USER)))
+ .thenReturn(true);
+ when(queue.hasAccess(eq(QueueACL.ADMINISTER_QUEUE), eq(BAD_USER)))
+ .thenReturn(false);
+ when(scheduler.getQueue(eq(queueName))).thenReturn(queue);
+ }
+ @Test
+ public void testDefaultPolicy() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, GOOD_USER.getShortUserName());
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ DefaultConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, null));
+ assertFalse(policy.isMutationAllowed(BAD_USER, null));
+ }
+
+ @Test
+ public void testQueueAdminBasedPolicy() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.a", EMPTY_MAP);
+ updateInfo.getUpdateQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyAddNestedQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Add root.b.b1.b11. Should check ACL of root.b queue.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ QueueConfigInfo configInfo = new QueueConfigInfo("root.b.b2.b21", EMPTY_MAP);
+ updateInfo.getAddQueueInfo().add(configInfo);
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+
+ @Test
+ public void testQueueAdminPolicyRemoveQueue() {
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS,
+ QueueAdminConfigurationMutationACLPolicy.class,
+ ConfigurationMutationACLPolicy.class);
+ policy = ConfigurationMutationACLPolicyFactory.getPolicy(conf);
+ policy.init(conf, rmContext);
+ // Remove root.b.b1.
+ QueueConfigsUpdateInfo updateInfo = new QueueConfigsUpdateInfo();
+ updateInfo.getRemoveQueueInfo().add("root.b.b1");
+ assertTrue(policy.isMutationAllowed(GOOD_USER, updateInfo));
+ assertFalse(policy.isMutationAllowed(BAD_USER, updateInfo));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f3fce9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 254da31..13229b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.QueueConfigsUpdateInfo;
import org.junit.Before;
import org.junit.Test;
@@ -43,22 +47,34 @@ public class TestMutableCSConfigurationProvider {
private MutableCSConfigurationProvider confProvider;
private RMContext rmContext;
- private Map<String, String> goodUpdate;
- private Map<String, String> badUpdate;
+ private QueueConfigsUpdateInfo goodUpdate;
+ private QueueConfigsUpdateInfo badUpdate;
private CapacityScheduler cs;
- private static final String TEST_USER = "testUser";
+ private static final UserGroupInformation TEST_USER = UserGroupInformation
+ .createUserForTesting("testUser", new String[] {});
@Before
public void setUp() {
cs = mock(CapacityScheduler.class);
rmContext = mock(RMContext.class);
when(rmContext.getScheduler()).thenReturn(cs);
+ when(cs.getConfiguration()).thenReturn(
+ new CapacitySchedulerConfiguration());
confProvider = new MutableCSConfigurationProvider(rmContext);
- goodUpdate = new HashMap<>();
- goodUpdate.put("goodKey", "goodVal");
- badUpdate = new HashMap<>();
- badUpdate.put("badKey", "badVal");
+ goodUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> goodUpdateMap = new HashMap<>();
+ goodUpdateMap.put("goodKey", "goodVal");
+ QueueConfigInfo goodUpdateInfo = new
+ QueueConfigInfo("root.a", goodUpdateMap);
+ goodUpdate.getUpdateQueueInfo().add(goodUpdateInfo);
+
+ badUpdate = new QueueConfigsUpdateInfo();
+ Map<String, String> badUpdateMap = new HashMap<>();
+ badUpdateMap.put("badKey", "badVal");
+ QueueConfigInfo badUpdateInfo = new
+ QueueConfigInfo("root.a", badUpdateMap);
+ badUpdate.getUpdateQueueInfo().add(badUpdateInfo);
}
@Test
@@ -66,15 +82,16 @@ public class TestMutableCSConfigurationProvider {
Configuration conf = new Configuration();
confProvider.init(conf);
assertNull(confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
doNothing().when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
confProvider.mutateConfiguration(TEST_USER, goodUpdate);
assertEquals("goodVal", confProvider.loadConfiguration(conf)
- .get("goodKey"));
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class),
any(RMContext.class));
try {
@@ -82,6 +99,7 @@ public class TestMutableCSConfigurationProvider {
} catch (IOException e) {
// Expected exception.
}
- assertNull(confProvider.loadConfiguration(conf).get("badKey"));
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-5951. Changes to allow
CapacityScheduler to use configuration store
Posted by jh...@apache.org.
YARN-5951. Changes to allow CapacityScheduler to use configuration store
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae0f61e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae0f61e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae0f61e3
Branch: refs/heads/YARN-5734
Commit: ae0f61e34321ca8a6da6dafa39e86dd29039221d
Parents: 2b08a1f
Author: Jonathan Hung <jh...@linkedin.com>
Authored: Mon Jan 30 19:03:48 2017 -0800
Committer: Jonathan Hung <jh...@linkedin.com>
Committed: Fri Oct 6 16:54:52 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 37 +++++------
.../CapacitySchedulerConfiguration.java | 9 +++
.../capacity/conf/CSConfigurationProvider.java | 46 ++++++++++++++
.../conf/FileBasedCSConfigurationProvider.java | 67 ++++++++++++++++++++
.../scheduler/capacity/conf/package-info.java | 29 +++++++++
.../capacity/TestCapacityScheduler.java | 4 +-
6 files changed, 170 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 7f50272..e74cbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.EnumSet;
@@ -103,6 +102,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.Activi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.AllocationState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.CSConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.FileBasedCSConfigurationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.KillableContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
@@ -167,6 +168,8 @@ public class CapacityScheduler extends
private int maxAssignPerHeartbeat;
+ private CSConfigurationProvider csConfProvider;
+
@Override
public void setConf(Configuration conf) {
yarnConf = conf;
@@ -289,7 +292,18 @@ public class CapacityScheduler extends
IOException {
try {
writeLock.lock();
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ String confProviderStr = configuration.get(
+ CapacitySchedulerConfiguration.CS_CONF_PROVIDER,
+ CapacitySchedulerConfiguration.DEFAULT_CS_CONF_PROVIDER);
+ if (confProviderStr.equals(
+ CapacitySchedulerConfiguration.FILE_CS_CONF_PROVIDER)) {
+ this.csConfProvider = new FileBasedCSConfigurationProvider(rmContext);
+ } else {
+ throw new IOException("Invalid CS configuration provider: " +
+ confProviderStr);
+ }
+ this.csConfProvider.init(configuration);
+ this.conf = this.csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
this.minimumAllocation = super.getMinimumAllocation();
initMaximumResourceCapability(super.getMaximumAllocation());
@@ -399,7 +413,7 @@ public class CapacityScheduler extends
writeLock.lock();
Configuration configuration = new Configuration(newConf);
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = loadCapacitySchedulerConfiguration(configuration);
+ this.conf = csConfProvider.loadConfiguration(configuration);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
@@ -1831,23 +1845,6 @@ public class CapacityScheduler extends
return true;
}
- private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
- Configuration configuration) throws IOException {
- try {
- InputStream CSInputStream =
- this.rmContext.getConfigurationProvider()
- .getConfigurationInputStream(configuration,
- YarnConfiguration.CS_CONFIGURATION_FILE);
- if (CSInputStream != null) {
- configuration.addResource(CSInputStream);
- return new CapacitySchedulerConfiguration(configuration, false);
- }
- return new CapacitySchedulerConfiguration(configuration, true);
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
private String getDefaultReservationQueueName(String planQueueName) {
return planQueueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3a519ec..3821e24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -315,6 +315,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
+
+ public static final String CS_CONF_PROVIDER = PREFIX
+ + "configuration.provider";
+
+ @Private
+ public static final String FILE_CS_CONF_PROVIDER = "file";
+
+ @Private
+ public static final String DEFAULT_CS_CONF_PROVIDER = FILE_CS_CONF_PROVIDER;
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
new file mode 100644
index 0000000..c9984ac
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/CSConfigurationProvider.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+
+/**
+ * Configuration provider for {@link CapacityScheduler}.
+ */
+public interface CSConfigurationProvider {
+
+ /**
+ * Initialize the configuration provider with given conf.
+ * @param conf configuration to initialize with
+ */
+ void init(Configuration conf);
+
+ /**
+ * Loads capacity scheduler configuration object.
+ * @param conf initial bootstrap configuration
+ * @return CS configuration
+ * @throws IOException if fail to retrieve configuration
+ */
+ CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
new file mode 100644
index 0000000..51c64fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FileBasedCSConfigurationProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * {@link CapacityScheduler} configuration provider based on local
+ * {@code capacity-scheduler.xml} file.
+ */
+public class FileBasedCSConfigurationProvider implements
+ CSConfigurationProvider {
+
+ private RMContext rmContext;
+
+ /**
+ * Construct file based CS configuration provider with given context.
+ * @param rmContext the RM context
+ */
+ public FileBasedCSConfigurationProvider(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void init(Configuration conf) {}
+
+ @Override
+ public CapacitySchedulerConfiguration loadConfiguration(Configuration conf)
+ throws IOException {
+ try {
+ InputStream csInputStream =
+ this.rmContext.getConfigurationProvider()
+ .getConfigurationInputStream(conf,
+ YarnConfiguration.CS_CONFIGURATION_FILE);
+ if (csInputStream != null) {
+ conf.addResource(csInputStream);
+ return new CapacitySchedulerConfiguration(conf, false);
+ }
+ return new CapacitySchedulerConfiguration(conf, true);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
new file mode 100644
index 0000000..08d0522
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package
+ * org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf
+ * contains classes related to capacity scheduler configuration management.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae0f61e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8440519..1dea4ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -241,13 +241,13 @@ public class TestCapacityScheduler {
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
- ResourceScheduler scheduler = new CapacityScheduler();
+ CapacityScheduler scheduler = new CapacityScheduler();
scheduler.setRMContext(resourceManager.getRMContext());
Configuration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
try {
- scheduler.reinitialize(conf, mockContext);
+ scheduler.init(conf);
fail("Exception is expected because the min memory allocation is" +
" larger than the max memory allocation.");
} catch (YarnRuntimeException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-7219. Make
AllocateRequestProto compatible with branch-2/branch-2.8. (rchiang)
Posted by jh...@apache.org.
YARN-7219. Make AllocateRequestProto compatible with branch-2/branch-2.8. (rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67e12169
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67e12169
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67e12169
Branch: refs/heads/YARN-5734
Commit: 67e12169b0ae8f3a9edb6511d3ea7ffd57b82e75
Parents: c9b525b
Author: Ray Chiang <rc...@apache.org>
Authored: Tue Oct 3 14:57:56 2017 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Tue Oct 3 15:06:07 2017 -0700
----------------------------------------------------------------------
.../hadoop-yarn-api/src/main/proto/yarn_service_protos.proto | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67e12169/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 3da4ee7..54a3c69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -88,7 +88,7 @@ message AllocateRequestProto {
optional ResourceBlacklistRequestProto blacklist_request = 3;
optional int32 response_id = 4;
optional float progress = 5;
- repeated UpdateContainerRequestProto update_requests = 6;
+ repeated UpdateContainerRequestProto update_requests = 7;
}
message NMTokenProto {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-7207. Cache the RM proxy
server address. (Yufei Gu)
Posted by jh...@apache.org.
YARN-7207. Cache the RM proxy server address. (Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72d22b75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72d22b75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72d22b75
Branch: refs/heads/YARN-5734
Commit: 72d22b753abde4d07a727479d3f3d5d84d5dd6b2
Parents: 572cdb5
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 6 09:45:10 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 6 09:49:11 2017 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/RMContext.java | 2 ++
.../server/resourcemanager/RMContextImpl.java | 37 ++++++++++++++++++--
.../server/resourcemanager/ResourceManager.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 20 +----------
.../rmapp/attempt/RMAppAttemptImpl.java | 23 ++----------
.../attempt/TestRMAppAttemptTransitions.java | 20 +++--------
6 files changed, 45 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 6df3135..ec94030 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -164,4 +164,6 @@ public interface RMContext extends ApplicationMasterServiceContext {
ResourceProfilesManager getResourceProfilesManager();
void setResourceProfilesManager(ResourceProfilesManager mgr);
+
+ String getAppProxyUrl(Configuration conf, ApplicationId applicationId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 921e5d5..80a9109 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -53,9 +57,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
/**
* RMContextImpl class holds two services context.
@@ -71,6 +77,8 @@ import com.google.common.annotations.VisibleForTesting;
*/
public class RMContextImpl implements RMContext {
+ private static final Log LOG = LogFactory.getLog(RMContextImpl.class);
+ private static final String UNAVAILABLE = "N/A";
/**
* RM service contexts which runs through out RM life span. These are created
* once during start of RM.
@@ -85,6 +93,8 @@ public class RMContextImpl implements RMContext {
private ResourceProfilesManager resourceProfilesManager;
+ private String proxyHostAndPort = null;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -118,7 +128,7 @@ public class RMContextImpl implements RMContext {
ConfigurationProvider provider = new LocalConfigurationProvider();
setConfigurationProvider(provider);
}
-
+
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
@@ -527,7 +537,7 @@ public class RMContextImpl implements RMContext {
public PlacementManager getQueuePlacementManager() {
return this.activeServiceContext.getQueuePlacementManager();
}
-
+
@Override
public void setQueuePlacementManager(PlacementManager placementMgr) {
this.activeServiceContext.setQueuePlacementManager(placementMgr);
@@ -559,6 +569,29 @@ public class RMContextImpl implements RMContext {
return this.resourceProfilesManager;
}
+ String getProxyHostAndPort(Configuration conf) {
+ if (proxyHostAndPort == null) {
+ proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
+ }
+ return proxyHostAndPort;
+ }
+
+ @Override
+ public String getAppProxyUrl(Configuration conf, ApplicationId applicationId)
+ {
+ try {
+ final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme,
+ getProxyHostAndPort(conf));
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
+ return result.toASCIIString();
+ } catch(URISyntaxException e) {
+ LOG.warn("Could not generate default proxy tracking URL for " +
+ applicationId);
+ return UNAVAILABLE;
+ }
+ }
+
@Override
public void setResourceProfilesManager(ResourceProfilesManager mgr) {
this.resourceProfilesManager = mgr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index e53a42c..fd95ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -1077,7 +1077,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
.withCSRFProtection(YarnConfiguration.RM_CSRF_PREFIX)
.withXFSProtection(YarnConfiguration.RM_XFS_PREFIX)
.at(webAppAddress);
- String proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
+ String proxyHostAndPort = rmContext.getProxyHostAndPort(conf);
if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf).
equals(proxyHostAndPort)) {
if (HAUtil.isHAEnabled(conf)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 09381f9..cfb8a74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -103,7 +101,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSch
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -113,7 +110,6 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -707,7 +703,7 @@ public class RMAppImpl implements RMApp, Recoverable {
float progress = 0.0f;
org.apache.hadoop.yarn.api.records.Token amrmToken = null;
if (allowAccess) {
- trackingUrl = getDefaultProxyTrackingUrl();
+ trackingUrl = rmContext.getAppProxyUrl(conf, applicationId);
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
@@ -797,20 +793,6 @@ public class RMAppImpl implements RMApp, Recoverable {
}
}
- private String getDefaultProxyTrackingUrl() {
- try {
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
- return result.toASCIIString();
- } catch (URISyntaxException e) {
- LOG.warn("Could not generate default proxy tracking URL for "
- + applicationId);
- return UNAVAILABLE;
- }
- }
-
@Override
public long getFinishTime() {
this.readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 1b4d613..4997bc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -521,7 +519,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
- this.proxiedTrackingUrl = generateProxyUriWithScheme();
+ this.proxiedTrackingUrl = rmContext.getAppProxyUrl(conf,
+ appAttemptId.getApplicationId());
this.stateMachine = stateMachineFactory.make(this);
this.attemptMetrics =
@@ -654,24 +653,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
- private String generateProxyUriWithScheme() {
- this.readLock.lock();
- try {
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
- applicationAttemptId.getApplicationId());
- return result.toASCIIString();
- } catch (URISyntaxException e) {
- LOG.warn("Could not proxify the uri for "
- + applicationAttemptId.getApplicationId(), e);
- return null;
- } finally {
- this.readLock.unlock();
- }
- }
-
private void setTrackingUrlToRMAppPage(RMAppAttemptState stateToBeStored) {
originalTrackingUrl = pjoin(
WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72d22b75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index f6406ff..9dd5703 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -35,8 +35,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -113,7 +111,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
@@ -344,19 +341,10 @@ public class TestRMAppAttemptTransitions {
((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop();
}
-
private String getProxyUrl(RMAppAttempt appAttempt) {
- String url = null;
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- try {
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri, appAttempt
- .getAppAttemptId().getApplicationId());
- url = result.toASCIIString();
- } catch (URISyntaxException ex) {
- Assert.fail();
- }
+ String url = rmContext.getAppProxyUrl(conf,
+ appAttempt.getAppAttemptId().getApplicationId());
+ Assert.assertNotEquals("N/A", url);
return url;
}
@@ -552,7 +540,7 @@ public class TestRMAppAttemptTransitions {
if (unmanagedAM) {
verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
- assertEquals(getProxyUrl(applicationAttempt),
+ assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
}
// TODO - need to add more checks relevant to this state
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDFS-12420. Add an option to
disallow 'namenode format -force'. Contributed by Ajay Kumar.
Posted by jh...@apache.org.
HDFS-12420. Add an option to disallow 'namenode format -force'. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6942cbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6942cbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6942cbe
Branch: refs/heads/YARN-5734
Commit: b6942cbe9b8c9469e8c2b64c3268d671f5a43e75
Parents: d0046bb
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Oct 5 15:26:52 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Oct 5 15:26:52 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++
.../hadoop/hdfs/server/namenode/NameNode.java | 16 +++++++++
.../namenode/NameNodeFormatException.java | 37 ++++++++++++++++++++
.../src/main/resources/hdfs-default.xml | 10 ++++++
.../src/site/markdown/HDFSCommands.md | 2 +-
.../hdfs/server/namenode/TestClusterId.java | 34 ++++++++++++++++++
6 files changed, 101 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b4842f9..f2f8730 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -270,6 +270,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.posix.acl.inheritance.enabled";
public static final boolean
DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = true;
+ public static final String DFS_REFORMAT_DISABLED = "dfs.reformat.disabled";
+ public static final boolean DFS_REFORMAT_DISABLED_DEFAULT = false;
+
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 79bbbc5..32b873b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.ha.ActiveState;
import org.apache.hadoop.hdfs.server.namenode.ha.BootstrapStandby;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -1146,6 +1147,21 @@ public class NameNode extends ReconfigurableBase implements
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
fsImage.getEditLog().initJournalsForWrite();
+ // Abort NameNode format if reformat is disabled and if
+ // meta-dir already exists
+ if (conf.getBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED,
+ DFSConfigKeys.DFS_REFORMAT_DISABLED_DEFAULT)) {
+ force = false;
+ isInteractive = false;
+ for (StorageDirectory sd : fsImage.storage.dirIterable(null)) {
+ if (sd.hasSomeData()) {
+ throw new NameNodeFormatException(
+ "NameNode format aborted as reformat is disabled for "
+ + "this cluster.");
+ }
+ }
+ }
+
if (!fsImage.confirmFormat(force, isInteractive)) {
return true; // aborted
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
new file mode 100644
index 0000000..858d6e1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeFormatException.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Thrown when NameNode format fails.
+ */
+@InterfaceAudience.Private
+public class NameNodeFormatException extends IOException {
+
+ private static final long serialVersionUID = 1L;
+
+ public NameNodeFormatException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public NameNodeFormatException(String message) {
+ super(message);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4e4ee33..8a60b80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4640,4 +4640,14 @@
</description>
</property>
+ <property>
+ <name>dfs.reformat.disabled</name>
+ <value>false</value>
+ <description>
+ Disable reformat of NameNode. If it's value is set to "true"
+ and metadata directories already exist then attempt to format NameNode
+ will throw NameNodeFormatException.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 322609a..c5f80d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -532,7 +532,7 @@ Usage:
|:---- |:---- |
| `-backup` | Start backup node. |
| `-checkpoint` | Start checkpoint node. |
-| `-format` `[-clusterid cid]` `[-force]` `[-nonInteractive]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. -force option formats if the name directory exists. -nonInteractive option aborts if the name directory exists, unless -force option is specified. |
+| `-format` `[-clusterid cid]` | Formats the specified NameNode. It starts the NameNode, formats it and then shut it down. Will throw NameNodeFormatException if name dir already exist and if reformat is disabled for cluster. |
| `-upgrade` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Namenode should be started with upgrade option after the distribution of new Hadoop version. |
| `-upgradeOnly` `[-clusterid cid]` [`-renameReserved` \<k-v pairs\>] | Upgrade the specified NameNode and then shutdown it. |
| `-rollback` | Rollback the NameNode to the previous version. This should be used after stopping the cluster and distributing the old Hadoop version. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6942cbe/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
index fa3399b..d5a3948 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
@@ -39,9 +39,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ExitUtil.ExitException;
@@ -452,4 +456,34 @@ public class TestClusterId {
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
+
+ /**
+ * Test NameNode format failure when reformat is disabled and metadata
+ * directories exist.
+ */
+ @Test
+ public void testNNFormatFailure() throws Exception {
+ NameNode.initMetrics(config, NamenodeRole.NAMENODE);
+ DFSTestUtil.formatNameNode(config);
+ config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
+ // Call to NameNode format will fail as name dir is not empty
+ try {
+ NameNode.format(config);
+ fail("NN format should fail.");
+ } catch (NameNodeFormatException e) {
+ GenericTestUtils.assertExceptionContains("NameNode format aborted as "
+ + "reformat is disabled for this cluster", e);
+ }
+ }
+
+ /**
+ * Test NameNode format when reformat is disabled and metadata directories do
+ * not exist.
+ */
+ @Test
+ public void testNNFormatSuccess() throws Exception {
+ NameNode.initMetrics(config, NamenodeRole.NAMENODE);
+ config.setBoolean(DFSConfigKeys.DFS_REFORMAT_DISABLED, true);
+ DFSTestUtil.formatNameNode(config);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-2162. Add ability in Fair
Scheduler to optionally configure maxResources in terms of percentage. (Yufei
Gu)
Posted by jh...@apache.org.
YARN-2162. Add ability in Fair Scheduler to optionally configure maxResources in terms of percentage. (Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99292adc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99292adc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99292adc
Branch: refs/heads/YARN-5734
Commit: 99292adcefdc6b8f280b8e100605fb39f755c38a
Parents: 25f31d9
Author: Yufei Gu <yu...@apache.org>
Authored: Thu Oct 5 23:51:58 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Thu Oct 5 23:51:58 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 44 ++++-------
.../fair/AllocationFileLoaderService.java | 47 +++++++-----
.../scheduler/fair/FSLeafQueue.java | 6 +-
.../scheduler/fair/FSParentQueue.java | 6 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 33 +++++---
.../scheduler/fair/FairScheduler.java | 3 +-
.../fair/FairSchedulerConfiguration.java | 54 +++++++++++--
.../scheduler/fair/QueueManager.java | 3 +-
.../fair/TestAllocationFileLoaderService.java | 22 +++---
.../scheduler/fair/TestFSLeafQueue.java | 2 +-
.../scheduler/fair/TestFSParentQueue.java | 3 +
.../scheduler/fair/TestFairScheduler.java | 15 ++--
.../fair/TestFairSchedulerConfiguration.java | 81 ++++++++++++++++----
.../fair/TestMaxRunningAppsEnforcer.java | 5 +-
.../scheduler/fair/TestQueueManager.java | 5 +-
.../webapp/dao/TestFairSchedulerQueueInfo.java | 3 +
.../src/site/markdown/FairScheduler.md | 4 +-
17 files changed, 226 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 7bd6959..3505bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
@@ -40,15 +38,13 @@ import com.google.common.annotations.VisibleForTesting;
public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
- private static final ResourceCalculator RESOURCE_CALCULATOR =
- new DefaultResourceCalculator();
// Minimum resource allocation for each queue
private final Map<String, Resource> minQueueResources;
// Maximum amount of resources per queue
@VisibleForTesting
- final Map<String, Resource> maxQueueResources;
+ final Map<String, ConfigurableResource> maxQueueResources;
// Maximum amount of resources for each queue's ad hoc children
- private final Map<String, Resource> maxChildQueueResources;
+ private final Map<String, ConfigurableResource> maxChildQueueResources;
// Sharing weights for each queue
private final Map<String, Float> queueWeights;
@@ -60,7 +56,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
final Map<String, Integer> userMaxApps;
private final int userMaxAppsDefault;
private final int queueMaxAppsDefault;
- private final Resource queueMaxResourcesDefault;
+ private final ConfigurableResource queueMaxResourcesDefault;
// Maximum resource share for each leaf queue that can be used to run AMs
final Map<String, Float> queueMaxAMShares;
@@ -109,14 +105,14 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private final Set<String> nonPreemptableQueues;
public AllocationConfiguration(Map<String, Resource> minQueueResources,
- Map<String, Resource> maxQueueResources,
- Map<String, Resource> maxChildQueueResources,
+ Map<String, ConfigurableResource> maxQueueResources,
+ Map<String, ConfigurableResource> maxChildQueueResources,
Map<String, Integer> queueMaxApps,
Map<String, Integer> userMaxApps,
Map<String, Float> queueWeights,
Map<String, Float> queueMaxAMShares, int userMaxAppsDefault,
int queueMaxAppsDefault,
- Resource queueMaxResourcesDefault,
+ ConfigurableResource queueMaxResourcesDefault,
float queueMaxAMShareDefault,
Map<String, SchedulingPolicy> schedulingPolicies,
SchedulingPolicy defaultSchedulingPolicy,
@@ -165,7 +161,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
queueMaxAMShares = new HashMap<>();
userMaxAppsDefault = Integer.MAX_VALUE;
queueMaxAppsDefault = Integer.MAX_VALUE;
- queueMaxResourcesDefault = Resources.unbounded();
+ queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded());
queueMaxAMShareDefault = 0.5f;
queueAcls = new HashMap<>();
resAcls = new HashMap<>();
@@ -291,26 +287,18 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
/**
* Get the maximum resource allocation for the given queue. If the max in not
- * set, return the larger of the min and the default max.
+ * set, return the default max.
*
* @param queue the target queue's name
* @return the max allocation on this queue
*/
@VisibleForTesting
- Resource getMaxResources(String queue) {
- Resource maxQueueResource = maxQueueResources.get(queue);
+ ConfigurableResource getMaxResources(String queue) {
+ ConfigurableResource maxQueueResource = maxQueueResources.get(queue);
if (maxQueueResource == null) {
- Resource minQueueResource = minQueueResources.get(queue);
- if (minQueueResource != null &&
- Resources.greaterThan(RESOURCE_CALCULATOR, Resources.unbounded(),
- minQueueResource, queueMaxResourcesDefault)) {
- return minQueueResource;
- } else {
- return queueMaxResourcesDefault;
- }
- } else {
- return maxQueueResource;
+ maxQueueResource = queueMaxResourcesDefault;
}
+ return maxQueueResource;
}
/**
@@ -320,7 +308,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* @return the max allocation on this queue or null if not set
*/
@VisibleForTesting
- Resource getMaxChildResources(String queue) {
+ ConfigurableResource getMaxChildResources(String queue) {
return maxChildQueueResources.get(queue);
}
@@ -418,9 +406,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
queue.setMaxChildQueueResource(getMaxChildResources(name));
// Set queue metrics.
- queue.getMetrics().setMinShare(getMinResources(name));
- queue.getMetrics().setMaxShare(getMaxResources(name));
- queue.getMetrics().setMaxApps(getQueueMaxApps(name));
+ queue.getMetrics().setMinShare(queue.getMinShare());
+ queue.getMetrics().setMaxShare(queue.getMaxShare());
+ queue.getMetrics().setMaxApps(queue.getMaxRunningApps());
queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 4d918c1..597af94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -226,8 +226,8 @@ public class AllocationFileLoaderService extends AbstractService {
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
Map<String, Resource> minQueueResources = new HashMap<>();
- Map<String, Resource> maxQueueResources = new HashMap<>();
- Map<String, Resource> maxChildQueueResources = new HashMap<>();
+ Map<String, ConfigurableResource> maxQueueResources = new HashMap<>();
+ Map<String, ConfigurableResource> maxChildQueueResources = new HashMap<>();
Map<String, Integer> queueMaxApps = new HashMap<>();
Map<String, Integer> userMaxApps = new HashMap<>();
Map<String, Float> queueMaxAMShares = new HashMap<>();
@@ -244,7 +244,8 @@ public class AllocationFileLoaderService extends AbstractService {
Set<String> nonPreemptableQueues = new HashSet<>();
int userMaxAppsDefault = Integer.MAX_VALUE;
int queueMaxAppsDefault = Integer.MAX_VALUE;
- Resource queueMaxResourcesDefault = Resources.unbounded();
+ ConfigurableResource queueMaxResourcesDefault =
+ new ConfigurableResource(Resources.unbounded());
float queueMaxAMShareDefault = 0.5f;
long defaultFairSharePreemptionTimeout = Long.MAX_VALUE;
long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
@@ -305,7 +306,7 @@ public class AllocationFileLoaderService extends AbstractService {
}
} else if ("queueMaxResourcesDefault".equals(element.getTagName())) {
String text = ((Text)element.getFirstChild()).getData().trim();
- Resource val =
+ ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text);
queueMaxResourcesDefault = val;
} else if ("userMaxAppsDefault".equals(element.getTagName())) {
@@ -448,8 +449,8 @@ public class AllocationFileLoaderService extends AbstractService {
*/
private void loadQueue(String parentName, Element element,
Map<String, Resource> minQueueResources,
- Map<String, Resource> maxQueueResources,
- Map<String, Resource> maxChildQueueResources,
+ Map<String, ConfigurableResource> maxQueueResources,
+ Map<String, ConfigurableResource> maxChildQueueResources,
Map<String, Integer> queueMaxApps,
Map<String, Integer> userMaxApps,
Map<String, Float> queueMaxAMShares,
@@ -496,17 +497,17 @@ public class AllocationFileLoaderService extends AbstractService {
Element field = (Element) fieldNode;
if ("minResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- Resource val =
+ ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text);
- minQueueResources.put(queueName, val);
+ minQueueResources.put(queueName, val.getResource());
} else if ("maxResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- Resource val =
+ ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text);
maxQueueResources.put(queueName, val);
} else if ("maxChildResources".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- Resource val =
+ ConfigurableResource val =
FairSchedulerConfiguration.parseResourceConfigValue(text);
maxChildQueueResources.put(queueName, val);
} else if ("maxRunningApps".equals(field.getTagName())) {
@@ -606,14 +607,24 @@ public class AllocationFileLoaderService extends AbstractService {
queueAcls.put(queueName, acls);
resAcls.put(queueName, racls);
- if (maxQueueResources.containsKey(queueName) &&
- minQueueResources.containsKey(queueName)
- && !Resources.fitsIn(minQueueResources.get(queueName),
- maxQueueResources.get(queueName))) {
- LOG.warn(
- String.format("Queue %s has max resources %s less than "
- + "min resources %s", queueName, maxQueueResources.get(queueName),
- minQueueResources.get(queueName)));
+ checkMinAndMaxResource(minQueueResources, maxQueueResources, queueName);
+ }
+
+ private void checkMinAndMaxResource(Map<String, Resource> minResources,
+ Map<String, ConfigurableResource> maxResources, String queueName) {
+
+ ConfigurableResource maxConfigurableResource = maxResources.get(queueName);
+ Resource minResource = minResources.get(queueName);
+
+ if (maxConfigurableResource != null && minResource != null) {
+ Resource maxResource = maxConfigurableResource.getResource();
+
+ // check whether max resource is bigger or equals to min resource when max
+ // resource are absolute values
+ if (maxResource != null && !Resources.fitsIn(minResource, maxResource)) {
+ LOG.warn(String.format("Queue %s has max resources %s less than "
+ + "min resources %s", queueName, maxResource, minResource));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 89ad1e6..60477d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -332,10 +332,10 @@ public class FSLeafQueue extends FSQueue {
readLock.unlock();
}
// Cap demand to maxShare to limit allocation to maxShare
- demand = Resources.componentwiseMin(tmpDemand, maxShare);
+ demand = Resources.componentwiseMin(tmpDemand, getMaxShare());
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
- + "; the max is " + maxShare);
+ + "; the max is " + getMaxShare());
LOG.debug("The updated fairshare for " + getName() + " is "
+ getFairShare());
}
@@ -611,7 +611,7 @@ public class FSLeafQueue extends FSQueue {
", Policy: " + policy.getName() +
", FairShare: " + getFairShare() +
", SteadyFairShare: " + getSteadyFairShare() +
- ", MaxShare: " + maxShare +
+ ", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
", ResourceUsage: " + getResourceUsage() +
", Demand: " + getDemand() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 5b4e4dc..e42c9f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -150,13 +150,13 @@ public class FSParentQueue extends FSQueue {
}
}
// Cap demand to maxShare to limit allocation to maxShare
- demand = Resources.componentwiseMin(demand, maxShare);
+ demand = Resources.componentwiseMin(demand, getMaxShare());
} finally {
writeLock.unlock();
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand +
- "; the max is " + maxShare);
+ "; the max is " + getMaxShare());
}
}
@@ -300,7 +300,7 @@ public class FSParentQueue extends FSQueue {
", Policy: " + policy.getName() +
", FairShare: " + getFairShare() +
", SteadyFairShare: " + getSteadyFairShare() +
- ", MaxShare: " + maxShare +
+ ", MaxShare: " + getMaxShare() +
", MinShare: " + minShare +
", ResourceUsage: " + getResourceUsage() +
", Demand: " + getDemand() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 8ae3cb6..e0df480 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -71,9 +71,9 @@ public abstract class FSQueue implements Queue, Schedulable {
protected float weights;
protected Resource minShare;
- protected Resource maxShare;
+ private ConfigurableResource maxShare;
protected int maxRunningApps;
- protected Resource maxChildQueueResource;
+ private ConfigurableResource maxChildQueueResource;
// maxAMShare is a value between 0 and 1.
protected float maxAMShare;
@@ -105,7 +105,7 @@ public abstract class FSQueue implements Queue, Schedulable {
*
* @param recursive whether child queues should be reinitialized recursively
*/
- public void reinit(boolean recursive) {
+ public final void reinit(boolean recursive) {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.initFSQueue(this);
updatePreemptionVariables();
@@ -157,26 +157,35 @@ public abstract class FSQueue implements Queue, Schedulable {
return minShare;
}
- public void setMaxShare(Resource maxShare){
+ public void setMaxShare(ConfigurableResource maxShare){
this.maxShare = maxShare;
}
+ @Override
+ public Resource getMaxShare() {
+ Resource maxResource = maxShare.getResource(scheduler.getClusterResource());
+
+ // Max resource should be greater than or equal to min resource
+ Resource result = Resources.componentwiseMax(maxResource, minShare);
+
+ if (!Resources.equals(maxResource, result)) {
+ LOG.warn(String.format("Queue %s has max resources %s less than "
+ + "min resources %s", getName(), maxResource, minShare));
+ }
+ return result;
+ }
+
public Resource getReservedResource() {
reservedResource.setMemorySize(metrics.getReservedMB());
reservedResource.setVirtualCores(metrics.getReservedVirtualCores());
return reservedResource;
}
- @Override
- public Resource getMaxShare() {
- return maxShare;
- }
-
- public void setMaxChildQueueResource(Resource maxChildShare){
+ public void setMaxChildQueueResource(ConfigurableResource maxChildShare){
this.maxChildQueueResource = maxChildShare;
}
- public Resource getMaxChildQueueResource() {
+ public ConfigurableResource getMaxChildQueueResource() {
return maxChildQueueResource;
}
@@ -415,7 +424,7 @@ public abstract class FSQueue implements Queue, Schedulable {
+ " because it has reserved containers.");
}
return false;
- } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) {
+ } else if (!Resources.fitsIn(getResourceUsage(), getMaxShare())) {
if (LOG.isDebugEnabled()) {
LOG.debug("Assigning container failed on node '" + node.getNodeName()
+ " because queue resource usage is larger than MaxShare: "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 9cf2b2e..37f583e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1513,7 +1513,8 @@ public class FairScheduler extends
if ((queue.getParent() != null) &&
!configuredLeafQueues.contains(queue.getName()) &&
!configuredParentQueues.contains(queue.getName())) {
- Resource max = queue.getParent().getMaxChildQueueResource();
+ ConfigurableResource max = queue.getParent().
+ getMaxChildQueueResource();
if (max != null) {
queue.setMaxShare(max);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 8e8e37b..960299b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -287,19 +288,60 @@ public class FairSchedulerConfiguration extends Configuration {
*
* @throws AllocationConfigurationException
*/
- public static Resource parseResourceConfigValue(String val)
+ public static ConfigurableResource parseResourceConfigValue(String val)
throws AllocationConfigurationException {
+ ConfigurableResource configurableResource;
try {
val = StringUtils.toLowerCase(val);
- int memory = findResource(val, "mb");
- int vcores = findResource(val, "vcores");
- return BuilderUtils.newResource(memory, vcores);
+ if (val.contains("%")) {
+ configurableResource = new ConfigurableResource(
+ getResourcePercentage(val));
+ } else {
+ int memory = findResource(val, "mb");
+ int vcores = findResource(val, "vcores");
+ configurableResource = new ConfigurableResource(
+ BuilderUtils.newResource(memory, vcores));
+ }
} catch (AllocationConfigurationException ex) {
throw ex;
} catch (Exception ex) {
throw new AllocationConfigurationException(
"Error reading resource config", ex);
}
+ return configurableResource;
+ }
+
+ private static double[] getResourcePercentage(
+ String val) throws AllocationConfigurationException {
+ double[] resourcePercentage = new double[ResourceType.values().length];
+ String[] strings = val.split(",");
+ if (strings.length == 1) {
+ double percentage = findPercentage(strings[0], "");
+ for (int i = 0; i < ResourceType.values().length; i++) {
+ resourcePercentage[i] = percentage/100;
+ }
+ } else {
+ resourcePercentage[0] = findPercentage(val, "memory")/100;
+ resourcePercentage[1] = findPercentage(val, "cpu")/100;
+ }
+ return resourcePercentage;
+ }
+
+ private static double findPercentage(String val, String units)
+ throws AllocationConfigurationException {
+ final Pattern pattern =
+ Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units);
+ Matcher matcher = pattern.matcher(val);
+ if (!matcher.find()) {
+ if (units.equals("")) {
+ throw new AllocationConfigurationException("Invalid percentage: " +
+ val);
+ } else {
+ throw new AllocationConfigurationException("Missing resource: " +
+ units);
+ }
+ }
+ return Double.parseDouble(matcher.group(1));
}
public long getUpdateInterval() {
@@ -307,8 +349,8 @@ public class FairSchedulerConfiguration extends Configuration {
}
private static int findResource(String val, String units)
- throws AllocationConfigurationException {
- Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units);
+ throws AllocationConfigurationException {
+ final Pattern pattern = Pattern.compile("(\\d+)(\\.\\d*)?\\s*" + units);
Matcher matcher = pattern.matcher(val);
if (!matcher.find()) {
throw new AllocationConfigurationException("Missing resource: " + units);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index c08d13e..8734877 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -41,7 +41,6 @@ import org.xml.sax.SAXException;
import com.google.common.annotations.VisibleForTesting;
import java.util.Iterator;
import java.util.Set;
-import org.apache.hadoop.yarn.api.records.Resource;
/**
* Maintains a list of queues as well as scheduling parameters for each queue,
@@ -330,7 +329,7 @@ public class QueueManager {
!configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) {
// For ad hoc queues, set their max reource allocations based on
// their parents' default child settings.
- Resource maxChild = parent.getMaxChildQueueResource();
+ ConfigurableResource maxChild = parent.getMaxChildQueueResource();
if (maxChild != null) {
child.setMaxShare(maxChild);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index c8b9ad8..67b46f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -235,21 +235,21 @@ public class TestAllocationFileLoaderService {
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(2048, 10),
- queueConf.getMaxResources("root.queueA"));
+ queueConf.getMaxResources("root.queueA").getResource());
assertEquals(Resources.createResource(5120, 110),
- queueConf.getMaxResources("root.queueB"));
- assertEquals(Resources.createResource(5120, 0),
- queueConf.getMaxResources("root.queueC"));
+ queueConf.getMaxResources("root.queueB").getResource());
+ assertEquals(Resources.createResource(4096, 100),
+ queueConf.getMaxResources("root.queueC").getResource());
assertEquals(Resources.createResource(4096, 100),
- queueConf.getMaxResources("root.queueD"));
+ queueConf.getMaxResources("root.queueD").getResource());
assertEquals(Resources.createResource(4096, 100),
- queueConf.getMaxResources("root.queueE"));
+ queueConf.getMaxResources("root.queueE").getResource());
assertEquals(Resources.createResource(4096, 100),
- queueConf.getMaxResources("root.queueF"));
+ queueConf.getMaxResources("root.queueF").getResource());
assertEquals(Resources.createResource(4096, 100),
- queueConf.getMaxResources("root.queueG"));
+ queueConf.getMaxResources("root.queueG").getResource());
assertEquals(Resources.createResource(4096, 100),
- queueConf.getMaxResources("root.queueG.queueH"));
+ queueConf.getMaxResources("root.queueG.queueH").getResource());
assertEquals(Resources.createResource(1024, 0),
queueConf.getMinResources("root.queueA"));
@@ -279,9 +279,9 @@ public class TestAllocationFileLoaderService {
assertNull("Max child resources unexpectedly set for queue root.queueE",
queueConf.getMaxChildResources("root.queueE"));
assertEquals(Resources.createResource(2048, 64),
- queueConf.getMaxChildResources("root.queueF"));
+ queueConf.getMaxChildResources("root.queueF").getResource());
assertEquals(Resources.createResource(2048, 64),
- queueConf.getMaxChildResources("root.queueG"));
+ queueConf.getMaxChildResources("root.queueG").getResource());
assertNull("Max child resources unexpectedly set for "
+ "queue root.queueG.queueH",
queueConf.getMaxChildResources("root.queueG.queueH"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index b6b3f7e..0bba35d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -81,7 +81,7 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
String queueName = "root.queue1";
FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
- schedulable.setMaxShare(maxResource);
+ schedulable.setMaxShare(new ConfigurableResource(maxResource));
assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE);
assertEquals(schedulable.getMetrics().getSchedulingPolicy(),
SchedulingPolicy.DEFAULT_POLICY.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
index 3ae8f83..671b94b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Before;
import org.junit.Test;
@@ -42,6 +43,8 @@ public class TestFSParentQueue {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
when(scheduler.getConf()).thenReturn(conf);
+ when(scheduler.getResourceCalculator()).thenReturn(
+ new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
notEmptyQueues = new HashSet<FSQueue>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index d5b1fcc..6b2109d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4611,7 +4611,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
- queue2.setMaxShare(Resource.newInstance(1024, 1));
+ queue2.setMaxShare(
+ new ConfigurableResource(Resource.newInstance(1024, 1)));
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -5209,6 +5210,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.reinitialize(conf, resourceManager.getRMContext());
Resource maxResource = Resources.createResource(1024 * 8);
+ ConfigurableResource maxResourceConf =
+ new ConfigurableResource(maxResource);
FSAppAttempt app1 = mock(FSAppAttempt.class);
Mockito.when(app1.getDemand()).thenReturn(maxResource);
@@ -5220,15 +5223,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSLeafQueue aQueue =
new FSLeafQueue("root.queue1.a", scheduler, queue1);
- aQueue.setMaxShare(maxResource);
+ aQueue.setMaxShare(maxResourceConf);
aQueue.addApp(app1, true);
FSLeafQueue bQueue =
new FSLeafQueue("root.queue1.b", scheduler, queue1);
- bQueue.setMaxShare(maxResource);
+ bQueue.setMaxShare(maxResourceConf);
bQueue.addApp(app2, true);
- queue1.setMaxShare(maxResource);
+ queue1.setMaxShare(maxResourceConf);
queue1.addChildQueue(aQueue);
queue1.addChildQueue(bQueue);
@@ -5266,7 +5269,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FSLeafQueue child1 =
scheduler.getQueueManager().getLeafQueue("parent.child1", false);
Resource resource = Resource.newInstance(4 * GB, 4);
- child1.setMaxShare(resource);
+ child1.setMaxShare(new ConfigurableResource(resource));
FSAppAttempt app = mock(FSAppAttempt.class);
Mockito.when(app.getDemand()).thenReturn(resource);
Mockito.when(app.getResourceUsage()).thenReturn(resource);
@@ -5295,7 +5298,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
childQueueString, child1.dumpState());
FSParentQueue parent =
scheduler.getQueueManager().getParentQueue("parent", false);
- parent.setMaxShare(resource);
+ parent.setMaxShare(new ConfigurableResource(resource));
parent.updateDemand();
String parentQueueString = "{Name: root.parent,"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 8e7b666..999aaae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -20,11 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
import static org.junit.Assert.assertEquals;
-import java.io.File;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Test;
@@ -32,23 +28,59 @@ public class TestFairSchedulerConfiguration {
@Test
public void testParseResourceConfigValue() throws Exception {
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2 vcores, 1024 mb"));
+ parseResourceConfigValue("2 vcores, 1024 mb").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024 mb, 2 vcores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("2vcores,1024mb").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024mb,2vcores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024 mb, 2 vcores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores"));
+ parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2vcores,1024mb"));
+ parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024mb,2vcores"));
+ parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource());
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores"));
+ parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource());
+
+ Resource clusterResource = BuilderUtils.newResource(2048, 4);
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("50% memory, 50% cpu").
+ getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("50% Memory, 50% CpU").
+ getResource(clusterResource));
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 Mb, 2 vCores"));
+ parseResourceConfigValue("50%").getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 4),
+ parseResourceConfigValue("50% memory, 100% cpu").
+ getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 4),
+ parseResourceConfigValue(" 100% cpu, 50% memory").
+ getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 0),
+ parseResourceConfigValue("50% memory, 0% cpu").
+ getResource(clusterResource));
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024 mb, 2 vcores "));
+ parseResourceConfigValue("50 % memory, 50 % cpu").
+ getResource(clusterResource));
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024.3 mb, 2.35 vcores "));
+ parseResourceConfigValue("50%memory,50%cpu").
+ getResource(clusterResource));
assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024. mb, 2. vcores "));
+ parseResourceConfigValue(" 50 % memory, 50 % cpu ").
+ getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("50.% memory, 50.% cpu").
+ getResource(clusterResource));
+
+ clusterResource = BuilderUtils.newResource(1024 * 10, 4);
+ assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
+ parseResourceConfigValue("10.9% memory, 50.6% cpu").
+ getResource(clusterResource));
}
@Test(expected = AllocationConfigurationException.class)
@@ -70,5 +102,24 @@ public class TestFairSchedulerConfiguration {
public void testGibberish() throws Exception {
parseResourceConfigValue("1o24vc0res");
}
-
+
+ @Test(expected = AllocationConfigurationException.class)
+ public void testNoUnitsPercentage() throws Exception {
+ parseResourceConfigValue("95%, 50% memory");
+ }
+
+ @Test(expected = AllocationConfigurationException.class)
+ public void testInvalidNumPercentage() throws Exception {
+ parseResourceConfigValue("95A% cpu, 50% memory");
+ }
+
+ @Test(expected = AllocationConfigurationException.class)
+ public void testCpuPercentageMemoryAbsolute() throws Exception {
+ parseResourceConfigValue("50% cpu, 1024 mb");
+ }
+
+ @Test(expected = AllocationConfigurationException.class)
+ public void testMemoryPercentageCpuAbsolute() throws Exception {
+ parseResourceConfigValue("50% memory, 2 vcores");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
index f77df1e..964ecf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Before;
import org.junit.Test;
@@ -55,7 +56,9 @@ public class TestMaxRunningAppsEnforcer {
AllocationConfiguration allocConf = new AllocationConfiguration(
conf);
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
-
+ when(scheduler.getResourceCalculator()).thenReturn(
+ new DefaultResourceCalculator());
+
queueManager = new QueueManager(scheduler);
queueManager.initialize(conf);
userMaxApps = allocConf.userMaxApps;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 05ab11c..eb2d402 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Before;
import org.junit.Test;
@@ -50,6 +51,8 @@ public class TestQueueManager {
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
when(scheduler.getConf()).thenReturn(conf);
+ when(scheduler.getResourceCalculator()).thenReturn(
+ new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
@@ -206,7 +209,7 @@ public class TestQueueManager {
queueManager.updateAllocationConfiguration(allocConf);
queueManager.getQueue("root.test").setMaxChildQueueResource(
- Resources.createResource(8192, 256));
+ new ConfigurableResource(Resources.createResource(8192, 256)));
FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
assertNotNull("Leaf queue root.test.childC was not created",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
index 67d7340..83b7e41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Assert;
import org.junit.Test;
@@ -43,6 +44,8 @@ public class TestFairSchedulerQueueInfo {
when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
when(scheduler.getConf()).thenReturn(conf);
when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1));
+ when(scheduler.getResourceCalculator()).thenReturn(
+ new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
QueueManager queueManager = new QueueManager(scheduler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99292adc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 2ccd5ff..64c450b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -85,9 +85,9 @@ The allocation file must be in XML format. The format contains five types of ele
* **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
- * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit.
+ * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit.
- * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly.
+ * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxRunningApps**: limit the number of apps from the queue to run at once
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org