You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/09 00:37:01 UTC
[1/2] git commit: SLIDER-149 test to work even without YARN-2571 setup
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 3a759d1ec -> 66957d4fc
SLIDER-149 test to work even without YARN-2571 setup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7887fd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7887fd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7887fd3c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7887fd3c01b67414c764d163e03953ad3dfcfc94
Parents: 3a759d1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 14:41:05 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 14:41:05 2014 -0700
----------------------------------------------------------------------
.../agent/standalone/TestStandaloneYarnRegistryAM.groovy | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7887fd3c/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 83774d2..84dcc50 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -74,7 +75,11 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
// get local binding
def registryOperations = microZKCluster.registryOperations
- registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+ try {
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+ } catch (PathNotFoundException e) {
+ log.warn(" RM is not apparently running registry services: {}", e, e)
+ }
ServiceLauncher<SliderClient> launcher
launcher = createStandaloneAM(clustername, true, false)
[2/2] git commit: SLIDER-149 AM cleans entries under AM registry
entry on initial startup
Posted by st...@apache.org.
SLIDER-149 AM cleans entries under AM registry entry on initial startup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66957d4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66957d4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66957d4f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 66957d4fcd428343f883959a6ecb927745b44fee
Parents: 7887fd3
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 15:36:50 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 15:36:50 2014 -0700
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 35 ++++++-----
.../YarnRegistryViewForProviders.java | 62 ++++++++++++++++----
.../providers/hbase/HBaseProviderService.java | 2 +-
3 files changed, 72 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8526207..a920a3e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -816,10 +816,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, actionQueues,
- liveContainers);
- sliderAMProvider.bind(stateForProviders, actionQueues,
- liveContainers);
+ providerService.bind(stateForProviders, actionQueues, liveContainers);
+ sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers);
// chaos monkey
maybeStartMonkey();
@@ -885,7 +883,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Ensure that the user is generated from a keytab and has no HDFS delegation
* tokens.
*
- * @param user
+ * @param user user to validate
* @throws SliderException
*/
protected void validateLoginUser(UserGroupInformation user)
@@ -939,16 +937,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
- appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT, agentWebApp.getPort());
+ appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
+ agentWebApp.getPort());
appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
- agentWebApp.getSecuredPort());
+ agentWebApp.getSecuredPort());
}
/**
* This registers the service instance and its external values
* @param instanceName name of this instance
* @param appid application ID
- * @throws Exception
+ * @throws IOException
*/
private void registerServiceInstance(String instanceName,
ApplicationId appid) throws IOException {
@@ -958,8 +957,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
URL amWebURI = new URL(appMasterTrackingUrl);
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
- String serviceName = SliderKeys.APP_TYPE;
- int id = appid.getId();
//Give the provider restricted access to the state, registry
setupInitialRegistryPaths();
@@ -998,10 +995,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// store for clients
log.info("Service Record \n{}", serviceRecord);
- yarnRegistryOperations.putService(service_user_name,
+ String sliderServicePath = yarnRegistryOperations.putService(service_user_name,
SliderKeys.APP_TYPE,
instanceName,
- serviceRecord);
+ serviceRecord, true);
+ boolean isFirstAttempt = 1 == appAttemptID.getAttemptId();
+ // delete the children in case there are any and this is an AM startup.
+ // just to make sure everything underneath is purged
+ if (isFirstAttempt) {
+ yarnRegistryOperations.deleteChildren(sliderServicePath, true);
+ }
yarnRegistryOperations.setSelfRegistration(serviceRecord);
// and a shorter lived binding to the app
@@ -1080,7 +1083,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
String cid = RegistryPathUtils.encodeYarnID(id.toString());
try {
- yarnRegistryOperations.rmComponent(cid);
+ yarnRegistryOperations.deleteComponent(cid);
} catch (IOException e) {
log.warn("Failed to delete container {} : {}", id, e, e);
}
@@ -1492,9 +1495,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onShutdownRequest() {
LOG_YARN.info("Shutdown Request received");
signalAMComplete(new ActionStopSlider("stop",
- EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- "Shutdown requested from RM"));
+ EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ "Shutdown requested from RM"));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index 4eb91ab..4104b16 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -18,6 +18,8 @@
package org.apache.slider.server.services.yarnregistry;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.registry.client.api.BindFlags;
import org.apache.hadoop.registry.client.api.RegistryOperations;
@@ -27,6 +29,11 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
public class YarnRegistryViewForProviders {
@@ -111,21 +118,28 @@ public class YarnRegistryViewForProviders {
}
/**
- * Add a service under a path
+ * Add a service under a path, optionally purging any history
* @param username user
* @param serviceClass service class to use under ~user
* @param serviceName name of the service
* @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
* @throws IOException
*/
- public void putService(String username,
+ public String putService(String username,
String serviceClass,
String serviceName,
- ServiceRecord record) throws IOException {
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
String path = RegistryUtils.servicePath(
username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
}
/**
@@ -133,22 +147,50 @@ public class YarnRegistryViewForProviders {
* @param serviceClass service class to use under ~user
* @param serviceName name of the service
* @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
* @throws IOException
*/
- public void putService(
+ public String putService(
String serviceClass,
String serviceName,
- ServiceRecord record) throws IOException {
- String path = RegistryUtils.servicePath(
- user, serviceClass, serviceName);
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ return putService(user, serviceClass, serviceName, record, deleteTreeFirst);
}
- public void rmComponent(String componentName) throws IOException {
+ /**
+ * Delete a component
+ * @param componentName component name
+ * @throws IOException
+ */
+ public void deleteComponent(String componentName) throws IOException {
String path = RegistryUtils.componentPath(
user, sliderServiceclass, instanceName,
componentName);
registryOperations.delete(path, false);
}
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ //ignored
+ }
+ Map<String, RegistryPathStatus> results =
+ new HashMap<String, RegistryPathStatus>();
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index a36ae70..88e1f9e 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -275,7 +275,7 @@ public class HBaseProviderService extends AbstractProviderService
log.warn("failed to create config URL: {}", e, e);
}
log.info("registering {}/{}", name, HBASE_SERVICE_TYPE);
- yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord);
+ yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord, true);
PublishedConfiguration publishedSite =
new PublishedConfiguration("HBase site", siteConf);