You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/10/09 18:46:09 UTC
[22/43] hadoop git commit: YARN-7207. Cache the RM proxy server
address. (Yufei Gu)
YARN-7207. Cache the RM proxy server address. (Yufei Gu)
(cherry picked from commit 72d22b753abde4d07a727479d3f3d5d84d5dd6b2)
# Conflicts:
# hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
# hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da0a9696
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da0a9696
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da0a9696
Branch: refs/heads/YARN-5734-branch-2
Commit: da0a96968d360edeaf66ef08cccf77dde3b2468d
Parents: c949e8e
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 6 09:45:10 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 6 09:56:24 2017 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/RMContext.java | 2 ++
.../server/resourcemanager/RMContextImpl.java | 37 ++++++++++++++++++--
.../server/resourcemanager/ResourceManager.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 20 +----------
.../rmapp/attempt/RMAppAttemptImpl.java | 23 ++----------
.../attempt/TestRMAppAttemptTransitions.java | 20 +++--------
6 files changed, 44 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 7dc7c92..02c06ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -153,4 +153,6 @@ public interface RMContext extends ApplicationMasterServiceContext {
String getHAZookeeperConnectionState();
ResourceManager getResourceManager();
+
+ String getAppProxyUrl(Configuration conf, ApplicationId applicationId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 527ad23..e04ff1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -51,7 +55,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -69,6 +75,8 @@ import com.google.common.annotations.VisibleForTesting;
*/
public class RMContextImpl implements RMContext {
+ private static final Log LOG = LogFactory.getLog(RMContextImpl.class);
+ private static final String UNAVAILABLE = "N/A";
/**
* RM service contexts which runs through out RM life span. These are created
* once during start of RM.
@@ -81,6 +89,8 @@ public class RMContextImpl implements RMContext {
*/
private RMActiveServiceContext activeServiceContext;
+ private String proxyHostAndPort = null;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -114,7 +124,7 @@ public class RMContextImpl implements RMContext {
ConfigurationProvider provider = new LocalConfigurationProvider();
setConfigurationProvider(provider);
}
-
+
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
@@ -511,7 +521,7 @@ public class RMContextImpl implements RMContext {
public PlacementManager getQueuePlacementManager() {
return this.activeServiceContext.getQueuePlacementManager();
}
-
+
@Override
public void setQueuePlacementManager(PlacementManager placementMgr) {
this.activeServiceContext.setQueuePlacementManager(placementMgr);
@@ -538,5 +548,26 @@ public class RMContextImpl implements RMContext {
return this.activeServiceContext.getRMAppLifetimeMonitor();
}
- // Note: Read java doc before adding any services over here.
+ String getProxyHostAndPort(Configuration conf) {
+ if (proxyHostAndPort == null) {
+ proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
+ }
+ return proxyHostAndPort;
+ }
+
+ @Override
+ public String getAppProxyUrl(Configuration conf, ApplicationId applicationId)
+ {
+ try {
+ final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme,
+ getProxyHostAndPort(conf));
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
+ return result.toASCIIString();
+ } catch(URISyntaxException e) {
+ LOG.warn("Could not generate default proxy tracking URL for " +
+ applicationId);
+ return UNAVAILABLE;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index aed295e..8ae0458 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -991,7 +991,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
.withCSRFProtection(YarnConfiguration.RM_CSRF_PREFIX)
.withXFSProtection(YarnConfiguration.RM_XFS_PREFIX)
.at(webAppAddress);
- String proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
+ String proxyHostAndPort = rmContext.getProxyHostAndPort(conf);
if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf).
equals(proxyHostAndPort)) {
if (HAUtil.isHAEnabled(conf)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 2f5a538..c29f244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.net.InetAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -100,7 +98,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -110,7 +107,6 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -649,7 +645,7 @@ public class RMAppImpl implements RMApp, Recoverable {
float progress = 0.0f;
org.apache.hadoop.yarn.api.records.Token amrmToken = null;
if (allowAccess) {
- trackingUrl = getDefaultProxyTrackingUrl();
+ trackingUrl = rmContext.getAppProxyUrl(conf, applicationId);
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
@@ -743,20 +739,6 @@ public class RMAppImpl implements RMApp, Recoverable {
}
}
- private String getDefaultProxyTrackingUrl() {
- try {
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
- return result.toASCIIString();
- } catch (URISyntaxException e) {
- LOG.warn("Could not generate default proxy tracking URL for "
- + applicationId);
- return UNAVAILABLE;
- }
- }
-
@Override
public long getFinishTime() {
this.readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 9124339..5bfb8b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -521,7 +519,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
- this.proxiedTrackingUrl = generateProxyUriWithScheme();
+ this.proxiedTrackingUrl = rmContext.getAppProxyUrl(conf,
+ appAttemptId.getApplicationId());
this.stateMachine = stateMachineFactory.make(this);
this.attemptMetrics =
@@ -654,24 +653,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
- private String generateProxyUriWithScheme() {
- this.readLock.lock();
- try {
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
- applicationAttemptId.getApplicationId());
- return result.toASCIIString();
- } catch (URISyntaxException e) {
- LOG.warn("Could not proxify the uri for "
- + applicationAttemptId.getApplicationId(), e);
- return null;
- } finally {
- this.readLock.unlock();
- }
- }
-
private void setTrackingUrlToRMAppPage(RMAppAttemptState stateToBeStored) {
originalTrackingUrl = pjoin(
WebAppUtils.getResolvedRMWebAppURLWithScheme(conf),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/da0a9696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index f6406ff..9dd5703 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -35,8 +35,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -113,7 +111,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManag
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
@@ -344,19 +341,10 @@ public class TestRMAppAttemptTransitions {
((AsyncDispatcher)this.spyRMContext.getDispatcher()).stop();
}
-
private String getProxyUrl(RMAppAttempt appAttempt) {
- String url = null;
- final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- try {
- String proxy = WebAppUtils.getProxyHostAndPort(conf);
- URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(null, proxyUri, appAttempt
- .getAppAttemptId().getApplicationId());
- url = result.toASCIIString();
- } catch (URISyntaxException ex) {
- Assert.fail();
- }
+ String url = rmContext.getAppProxyUrl(conf,
+ appAttempt.getAppAttemptId().getApplicationId());
+ Assert.assertNotEquals("N/A", url);
return url;
}
@@ -552,7 +540,7 @@ public class TestRMAppAttemptTransitions {
if (unmanagedAM) {
verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
- assertEquals(getProxyUrl(applicationAttempt),
+ assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
}
// TODO - need to add more checks relevant to this state
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org