You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2017/09/22 00:59:04 UTC
[43/50] [abbrv] hadoop git commit: YARN-6866. Minor clean-up and
fixes in anticipation of YARN-2915 merge with trunk. (Botong Huang via
Subru).
YARN-6866. Minor clean-up and fixes in anticipation of YARN-2915 merge with trunk. (Botong Huang via Subru).
(cherry picked from commit 40453879ec860819b080ccc82454480a436c0adc)
(cherry picked from commit 86f05c33125af3b435d6793551680ffa35047aa7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/049f7c84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/049f7c84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/049f7c84
Branch: refs/heads/branch-2
Commit: 049f7c84a1dae5576012c69043bb98d073c85368
Parents: 66980c0
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Jul 25 20:22:45 2017 -0700
Committer: Carlo Curino <cu...@apache.org>
Committed: Thu Sep 21 16:55:43 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 6 +++---
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
.../hadoop/yarn/conf/TestYarnConfigurationFields.java | 4 ++++
.../router/clientrm/FederationClientInterceptor.java | 13 +++----------
.../hadoop-yarn-site/src/site/markdown/Federation.md | 8 ++++----
5 files changed, 16 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/049f7c84/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d61dc4b..6b24a75 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -89,9 +89,9 @@
<apacheds.version>2.0.0-M15</apacheds.version>
<jcache.version>1.0.0</jcache.version>
- <ehcache.version>3.0.3</ehcache.version>
- <hikari.version>2.4.11</hikari.version>
- <mssql.version>6.1.0.jre7</mssql.version>
+ <ehcache.version>3.3.1</ehcache.version>
+ <hikari.version>2.4.12</hikari.version>
+ <mssql.version>6.2.1.jre7</mssql.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.7</javac.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/049f7c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 3600709..7adfdf1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2592,7 +2592,7 @@ public class YarnConfiguration extends Configuration {
ROUTER_PREFIX + "clientrm.";
public static final String ROUTER_CLIENTRM_ADDRESS =
- ROUTER_CLIENTRM_PREFIX + ".address";
+ ROUTER_CLIENTRM_PREFIX + "address";
public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050;
public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS =
"0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT;
@@ -2610,7 +2610,7 @@ public class YarnConfiguration extends Configuration {
public static final String ROUTER_RMADMIN_PREFIX = ROUTER_PREFIX + "rmadmin.";
public static final String ROUTER_RMADMIN_ADDRESS =
- ROUTER_RMADMIN_PREFIX + ".address";
+ ROUTER_RMADMIN_PREFIX + "address";
public static final int DEFAULT_ROUTER_RMADMIN_PORT = 8052;
public static final String DEFAULT_ROUTER_RMADMIN_ADDRESS =
"0.0.0.0:" + DEFAULT_ROUTER_RMADMIN_PORT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/049f7c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 910df9e..56fb578 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -79,6 +79,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.ROUTER_CLIENTRM_ADDRESS);
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.ROUTER_RMADMIN_ADDRESS);
// Federation policies configs to be ignored
configurationPropsToSkipCompare
http://git-wip-us.apache.org/repos/asf/hadoop/blob/049f7c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index ecf53ac..7268ebd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.router.clientrm;
import java.io.IOException;
-import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -88,10 +87,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRespo
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
import org.apache.hadoop.yarn.server.federation.policies.RouterPolicyFacade;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
@@ -175,14 +174,8 @@ public class FederationClientInterceptor
ApplicationClientProtocol clientRMProxy = null;
try {
- clientRMProxy =
- user.doAs(new PrivilegedExceptionAction<ApplicationClientProtocol>() {
- @Override
- public ApplicationClientProtocol run() throws Exception {
- return ClientRMProxy.createRMProxy(getConf(),
- ApplicationClientProtocol.class);
- }
- });
+ clientRMProxy = FederationProxyProviderUtil.createRMProxy(getConf(),
+ ApplicationClientProtocol.class, subClusterId, user);
} catch (Exception e) {
RouterServerUtil.logAndThrowException(
"Unable to create the interface to reach the SubCluster "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/049f7c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 79225b4..61cb77f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -209,9 +209,9 @@ Optional:
| Property | Example | Description |
|:---- |:---- |
|`yarn.router.hostname` | `0.0.0.0` | Router host name.
-|`yarn.router.clientrm.address` | `0.0.0.0:8032` | Router client address. |
+|`yarn.router.clientrm.address` | `0.0.0.0:8050` | Router client address. |
|`yarn.router.webapp.address` | `0.0.0.0:80` | Webapp address at the router. |
-|`yarn.router.admin.address` | `0.0.0.0:8033` | Admin address at the router. |
+|`yarn.router.admin.address` | `0.0.0.0:8052` | Admin address at the router. |
|`yarn.router.webapp.https.address` | `0.0.0.0:443` | Secure webapp address at the router. |
|`yarn.router.submit.retry` | `3` | The number of retries in the router before we give up. |
|`yarn.federation.statestore.max-connections` | `10` | This is the maximum number of parallel connections each Router makes to the state-store. |
@@ -227,7 +227,7 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
|:---- |:---- |
| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled.
|`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor.
-| `yarn.client.failover.proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
+| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
Optional:
@@ -242,7 +242,7 @@ Currently, the only supported implementation of the state-store is Microsoft SQL
Running a Sample Job
--------------------
-In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which said jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
+In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
| Property | Example | Description |
|:--- |:--- |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org