You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/09 00:53:16 UTC
[27/50] [abbrv] git commit: SLIDER-149;
in sync with registry code. HBase provider test working
SLIDER-149; in sync with registry code. HBase provider test working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46fb5dbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46fb5dbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46fb5dbd
Branch: refs/heads/develop
Commit: 46fb5dbdb35c5b072771a394ec339f1bd7e8f56c
Parents: 2154c75
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 26 14:18:06 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 26 14:18:06 2014 +0100
----------------------------------------------------------------------
.../registry/retrieve/RegistryRetriever.java | 43 +++++++++++++-------
.../server/appmaster/SliderAppMaster.java | 3 +-
.../YarnRegistryViewForProviders.java | 17 +++++---
.../TestStandaloneYarnRegistryAM.groovy | 2 +-
.../providers/hbase/HBaseProviderService.java | 12 ++++--
5 files changed, 51 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 1d11ef3..df1d675 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -25,8 +25,10 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.exceptions.RegistryIOException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.common.tools.SliderUtils;
@@ -68,25 +70,37 @@ public class RegistryRetriever {
this.internalConfigurationURL = internalConfigurationURL;
}
- public RegistryRetriever(ServiceRecord record) throws InvalidRecordException {
+ /**
+ * Retrieve from a service by locating the
+ * exported {@link CustomRegistryConstants#PUBLISHER_CONFIGURATIONS_API}
+ * and working off it.
+ * @param record service record
+ * @throws RegistryIOException the address type of the endpoint does
+ * not match that expected (i.e. not a list of URLs), missing endpoint...
+ */
+ public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
Endpoint internal = record.getInternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
- List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
- internal);
- if (addresses != null && !addresses.isEmpty()) {
- internalConfigurationURL = addresses.get(0);
- } else {
- internalConfigurationURL = "";
+ String url = null;
+ if (internal != null) {
+ List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
}
+ internalConfigurationURL = url;
Endpoint external = record.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-
- addresses = RegistryTypeUtils.retrieveAddressesUriType(external);
- if (addresses != null && !addresses.isEmpty()) {
- externalConfigurationURL = addresses.get(0);
- } else {
- externalConfigurationURL = "";
+ url = null;
+ if (external != null) {
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
}
+ externalConfigurationURL = url;
}
/**
@@ -166,7 +180,8 @@ public class RegistryRetriever {
@Override
public String toString() {
return super.toString()
- + " - external " + externalConfigurationURL;
+ + ": internal URL: \"" + internalConfigurationURL
+ + "\"; external \"" + externalConfigurationURL +"\"";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 2d3e5e4..2625d74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -901,7 +901,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
yarnRegistryOperations = new YarnRegistryViewForProviders(
registryOperations, service_user_name,
SliderKeys.APP_TYPE,
- instanceName);
+ instanceName,
+ appAttemptID);
providerService.bindToYarnRegistry(yarnRegistryOperations);
sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index d699279..e5b150a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -18,6 +18,7 @@
package org.apache.slider.server.services.yarnregistry;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
@@ -35,14 +36,23 @@ public class YarnRegistryViewForProviders {
private final String sliderServiceclass;
private final String instanceName;
+ private final ApplicationAttemptId applicationAttemptId;
private ServiceRecord selfRegistration;
public YarnRegistryViewForProviders(RegistryOperations registryOperations,
- String user, String sliderServiceclass, String instanceName) {
+ String user,
+ String sliderServiceclass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
this.registryOperations = registryOperations;
this.user = user;
this.sliderServiceclass = sliderServiceclass;
this.instanceName = instanceName;
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId;
}
public String getUser() {
@@ -100,7 +110,6 @@ public class YarnRegistryViewForProviders {
registryOperations.create(path, record, CreateFlags.OVERWRITE);
}
-
/**
* Add a service under a path
* @param username user
@@ -117,10 +126,8 @@ public class YarnRegistryViewForProviders {
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
-
}
-
/**
* Add a service under a path for the current user
* @param serviceClass service class to use under ~user
@@ -136,10 +143,8 @@ public class YarnRegistryViewForProviders {
user, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
-
}
-
public void rmComponent(String componentName) throws IOException {
String path = RegistryOperationUtils.componentPath(
user, sliderServiceclass, instanceName,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 4cd662b..d081676 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -139,7 +139,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
RegistryOperationsClient registryOperationsClient =
registryService as RegistryOperationsClient
try {
- def yarnRegistryDump = registryOperationsClient.dumpPath()
+ def yarnRegistryDump = registryOperationsClient.dumpPath(false)
log.info("yarn service registry: \n${yarnRegistryDump}\n")
} catch (IOException ignored) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 3c059d9..1760e72 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
@@ -254,16 +255,19 @@ public class HBaseProviderService extends AbstractProviderService
private void registerHBaseServiceEntry() throws IOException {
-
String name = amState.getApplicationName() ;
ServiceRecord serviceRecord = new ServiceRecord();
-
+ // bond lifespan to the application
+ serviceRecord.yarn_id = yarnRegistry.getApplicationAttemptId()
+ .getApplicationId().toString();
+ serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION;
try {
- URL configURL = new URL(amWebAPI, SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
+ URL configURL = new URL(amWebAPI,
+ SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.restEndpoint(
- CustomRegistryConstants.PUBLISHER_REST_API,
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
configURL.toURI()));
} catch (URISyntaxException e) {
log.warn("failed to create config URL: {}", e, e);