You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/12/11 16:03:08 UTC
incubator-slider git commit: SLIDER-907 AgentWebPagesIT#testAgentWeb
fails on wire encrypted clusters: publish proxied values from outset
Repository: incubator-slider
Updated Branches:
refs/heads/develop 87ed02cab -> 1742ab232
SLIDER-907 AgentWebPagesIT#testAgentWeb fails on wire encrypted clusters: publish proxied values from outset
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1742ab23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1742ab23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1742ab23
Branch: refs/heads/develop
Commit: 1742ab232ba65e95cb287b4f040def194effbf1f
Parents: 87ed02c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Dec 11 15:03:21 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Dec 11 15:03:21 2015 +0000
----------------------------------------------------------------------
.../slider/providers/ProviderService.java | 8 +++---
.../slideram/SliderAMProviderService.java | 6 ++---
.../server/appmaster/SliderAppMaster.java | 21 +++++++++++++--
.../server/appmaster/web/rest/RestPaths.java | 14 ++++++++--
.../TestStandaloneYarnRegistryAM.groovy | 28 ++++++++++++--------
5 files changed, 55 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 28cef01..1a18db6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -186,10 +186,10 @@ public interface ProviderService extends ProviderCore,
/**
* Prior to going live -register the initial service registry data
- * @param amWebURI
- * @param agentOpsURI
- * @param agentStatusURI
- * @param serviceRecord
+ * @param amWebURI URL to the AM. This may be proxied, so use relative paths
+ * @param agentOpsURI URI for agent operations. This will not be proxied
+ * @param agentStatusURI URI For agent status. Again: no proxy
+ * @param serviceRecord service record to build up
*/
void applyInitialRegistryDefinitions(URL amWebURI,
URL agentOpsURI,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index e382058..f0dab8c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -141,10 +141,10 @@ public class SliderAMProviderService extends AbstractProviderService implements
try {
- URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
- URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY );
+ URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT);
+ URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY);
- URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
+ URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER);
// Set the configurations URL.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index ef5290b..415a597 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -281,6 +282,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private String appMasterTrackingUrl = "";
+ /** Proxied app master URL (as retrieved from AM report at launch time) */
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+ private String appMasterProxiedUrl = "";
+
/** Application Attempt Id ( combination of attemptId and fail count )*/
private ApplicationAttemptId appAttemptID;
@@ -823,6 +828,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
+ // look up the application itself -this is needed to get the proxied
+ // URL of the AM, for registering endpoints.
+ // this call must be made after the AM has registered itself, obviously
+ ApplicationAttemptReport report = yarnClient.getApplicationAttemptReport(
+ appAttemptID);
+ appMasterProxiedUrl = report.getTrackingUrl();
+ if (SliderUtils.isUnset(appMasterProxiedUrl)) {
+ log.warn("Proxied URL is not set in application report");
+ appMasterProxiedUrl = appMasterTrackingUrl;
+ }
+
// extract container list
liveContainers = amRegistrationData.getContainersFromPreviousAttempts();
@@ -949,7 +965,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// log the YARN and web UIs
log.info("RM Webapp address {}",
serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));
- log.info("slider Webapp address {}", appMasterTrackingUrl);
+ log.info("Slider webapp address {} proxied at {}",
+ appMasterTrackingUrl, appMasterProxiedUrl);
// Start the Slider AM provider
sliderAMProvider.start();
@@ -1190,7 +1207,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// the registry is running, so register services
- URL amWebURI = new URL(appMasterTrackingUrl);
+ URL amWebURI = new URL(appMasterProxiedUrl);
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index b90eb62..ae9eb0f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -35,8 +35,10 @@ public class RestPaths {
* agent content root: {@value}
*/
public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
- public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
- public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
+ public static final String V1_SLIDER = "/v1/slider";
+ public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + V1_SLIDER;
+ public static final String RELATIVE_API = WS_CONTEXT + V1_SLIDER;
+ public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT + V1_SLIDER;
public static final String MANAGEMENT = "mgmt";
public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
@@ -49,6 +51,9 @@ public class RestPaths {
public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_MANAGEMENT;
+ public static final String RELATIVE_PATH_MANAGEMENT = RELATIVE_API
+ + SLIDER_SUBPATH_MANAGEMENT;
+
/**
* Agents: {@value}
*/
@@ -61,6 +66,9 @@ public class RestPaths {
public static final String SLIDER_PATH_PUBLISHER = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_PUBLISHER;
+ public static final String RELATIVE_PATH_PUBLISHER = RELATIVE_API
+ + SLIDER_SUBPATH_PUBLISHER;
+
/**
* Registry subpath: {@value}
*/
@@ -71,6 +79,8 @@ public class RestPaths {
*/
public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
+ public static final String RELATIVE_PATH_REGISTRY = RELATIVE_API
+ + SLIDER_SUBPATH_REGISTRY;
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1742ab23/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 139860a..56fb5f2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -18,10 +18,13 @@
package org.apache.slider.agent.standalone
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.types.AddressTypes
+import org.apache.hadoop.registry.client.types.ProtocolTypes
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -31,7 +34,6 @@ import org.apache.hadoop.registry.client.impl.RegistryOperationsClient
import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.exceptions.NotFoundException
@@ -59,7 +61,7 @@ import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
/**
* work with a YARN registry
*/
-//@CompileStatic
+@CompileStatic
@Slf4j
class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
@@ -165,15 +167,19 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
def externalEndpoints = serviceRecord.external;
assert externalEndpoints.size() > 0
-/*
-
- def am_ipc_protocol = AM_IPC_PROTOCOL
- serviceRecord.getExternalEndpoint(am_ipc_protocol)
- assert null != am_ipc_protocol;
-*/
assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ def publisherEPR = serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ assert publisherEPR.protocolType == ProtocolTypes.PROTOCOL_REST
+ assert publisherEPR.addressType == AddressTypes.ADDRESS_URI
+ def addr = publisherEPR.addresses[0][AddressTypes.ADDRESS_URI]
+ def proxiedURL = new URL(addr)
+ def appId = report.applicationId.toString()
+ // something like http://host:49257/proxy/application_1449844996855_0001/ws/v1/slider/publisher
+ String publisherPath = "/proxy/$appId/" + RestPaths.RELATIVE_PATH_PUBLISHER
+ assert proxiedURL.path == publisherPath
+
// internals
assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
@@ -323,8 +329,8 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
String confJSON = GET(yarnSitePublisher)
// log.info(confJSON)
- JsonSerDeser< PublishedConfiguration> confSerDeser =
- new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+ JsonSerDeser<PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<>(PublishedConfiguration)
publishedYarnSite = confSerDeser.fromJson(confJSON)
@@ -488,7 +494,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert LauncherExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
//look for a different user
- assertFailsWithException(SliderExitCodes.EXIT_NOT_FOUND, "") {
+ assertFailsWithException(LauncherExitCodes.EXIT_NOT_FOUND, "") {
def args = new ActionRegistryArgs(
name: clustername,
user: "unknown",