You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/09 00:52:50 UTC
[01/50] [abbrv] git commit: SLIDER-149 add PersistencePolicy
Repository: incubator-slider
Updated Branches:
refs/heads/develop 3b32b8f7d -> 2553f1573
SLIDER-149 add PersistencePolicy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c5fb4f04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c5fb4f04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c5fb4f04
Branch: refs/heads/develop
Commit: c5fb4f040bd0727426405815a6aba8542c07aa2d
Parents: 4cbc9fe
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 29 17:49:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 29 17:49:03 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c5fb4f04/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a2545ab..e80a362 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.server.services.ResourceManagerRegistryService;
@@ -165,9 +166,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_AGENT_CONTEXT_ROOT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
-
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -989,7 +987,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* Handler for {@link RegisterComponentInstance action}
- * Register/re-register a component (that is already in the app state
+ * Register/re-register an ephemeral container that is already in the app state
* @param id the component
* @param description
*/
@@ -1004,7 +1002,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
ServiceRecord container = new ServiceRecord(
cid,
- description);
+ description,
+ PersistencePolicies.EPHEMERAL);
try {
yarnRegistryOperations.putComponent(cid, container, true);
} catch (IOException e) {
[18/50] [abbrv] git commit: SLIDER-149 whitespace
Posted by st...@apache.org.
SLIDER-149 whitespace
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a06d5ed7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a06d5ed7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a06d5ed7
Branch: refs/heads/develop
Commit: a06d5ed72824adeb71cd225f718959e50d0401f6
Parents: db52ff5
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:53:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:53:26 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/SliderAppMaster.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a06d5ed7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1fd378b..fd7c360 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -327,14 +327,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private SliderRegistryService registry;
-
/**
* The YARN registry service
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RegistryOperationsService registryOperations;
-
/**
* Record of the max no. of cores allowed in this cluster
*/
[44/50] [abbrv] git commit: SLIDER-149: switch to the branch-2.6
version
Posted by st...@apache.org.
SLIDER-149: switch to the branch-2.6 version
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cc2a9752
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cc2a9752
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cc2a9752
Branch: refs/heads/develop
Commit: cc2a975223b76c5cc80b79b364825842a31f57b2
Parents: 9175aa7
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 10:37:27 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 10:37:27 2014 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cc2a9752/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d7b5711..e15632e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,7 +137,7 @@
<!--
core artifacts
-->
- <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
<accumulo.version>1.6.1</accumulo.version>
[10/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0eac88e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0eac88e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0eac88e
Branch: refs/heads/develop
Commit: b0eac88e009cf40339aed853c3a007ad5e75fbe5
Parents: f67b550 3511026
Author: Steve Loughran <st...@apache.org>
Authored: Mon Sep 15 10:28:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Sep 15 10:28:05 2014 +0100
----------------------------------------------------------------------
app-packages/storm/metainfo.xml | 2 +-
.../appmaster/model/mock/MockContainerId.groovy | 16 ++++++++++++++--
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[23/50] [abbrv] SLIDER-149 migration off curator-x-discovery: read
only REST API (work in progress)
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 68cac7b..e04f9cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -96,11 +96,6 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
assert instance != null
//switch to the slider ZK-based registry
-
- describe "service registry names"
- SliderRegistryService registry = client.registry
- def names = registry.getServiceTypes();
- dumpRegistryServiceTypes(names)
describe "service registry instance IDs"
def instanceIds = client.listRegisteredSliderInstances()
@@ -108,14 +103,6 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
log.info("number of instanceIds: ${instanceIds.size()}")
instanceIds.each { String it -> log.info(it) }
- describe "service registry slider instances"
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- client.listRegistryInstances()
- instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
- log.info svc.toString()
- }
- describe "end list service registry slider instances"
-
describe "Yarn registry"
def yarnRegistry = client.registryOperations
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index e88fbe8..65bd077 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -20,17 +20,18 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsClient
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
@@ -73,7 +74,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
// verify the cluster has the YARN reg service live
- def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
+ def rmRegistryService = miniCluster.getResourceManager(0).RMContext.registry
assert rmRegistryService
ServiceLauncher<SliderClient> launcher
@@ -124,48 +125,47 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
// sleep to allow registration to complete
sleep(5000)
-
-
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ RegistryOperationsClient registryOperationsClient =
+ registryService as RegistryOperationsClient
try {
- def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ def yarnRegistryDump = registryOperationsClient.dumpPath()
log.info("yarn service registry: \n${yarnRegistryDump}\n")
} catch (IOException ignored) {
}
-
- describe "service registry names"
- def registryService = client.registryOperations
def self = currentUser()
- RegistryPathStatus[] serviceTypes = registryService.list(userPath(self))
- dumpArray(serviceTypes)
+ List<RegistryPathStatus> serviceTypes = registryService.listFull(homePathForUser(self))
+ dumpCollection(serviceTypes)
def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
registryService,
- registryService.list(recordsPath))
+ registryService.listFull(recordsPath))
def serviceRecords = recordMap.values();
dumpCollection(serviceRecords)
assert serviceRecords.size() == 1
- def serviceInstance = serviceRecords[0]
- log.info(serviceInstance.toString())
+ def serviceRecord = serviceRecords[0]
+ log.info(serviceRecord.toString())
- assert 2 <= serviceInstance.external.size()
+ assert 2 <= serviceRecord.external.size()
// hit the registry web page
-
- def registryEndpoint = serviceInstance.getExternalEndpoint(
+ def registryEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.REGISTRY_REST_API)
assert registryEndpoint != null
def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
- describe("Registry WADL @ $registryURL")
- def publisherEndpoint = serviceInstance.getExternalEndpoint(
+ describe("Registry WADL @ $registryURL")
+ def publisherEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_REST_API)
def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
@@ -231,7 +231,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
describe "Registry Retrieval Class"
// retrieval
- RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
+ RegistryRetriever retriever = new RegistryRetriever(serviceRecord)
log.info retriever.toString()
assert retriever.hasConfigurations(true)
@@ -286,7 +286,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
registryArgs.name = "unknown"
try {
client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
+ } catch (UnknownApplicationInstanceException expected) {
// expected
}
@@ -296,7 +296,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
registryArgs.serviceType = "org-apache-hadoop"
try {
client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
+ } catch (UnknownApplicationInstanceException expected) {
// expected
}
@@ -311,7 +311,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
describe registryArgs.toString()
def listedInstance = client.actionRegistryListYarn(registryArgs)
- assert listedInstance[0].id == serviceInstance.id
+ assert listedInstance[0].id == serviceRecord.id
// listconf
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 7c5de08..07fa327 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -33,7 +33,6 @@ import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.launch.ContainerLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.actions.QueueAccess
@@ -46,7 +45,6 @@ import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse
import org.apache.slider.server.appmaster.web.rest.agent.Register
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
-import org.apache.slider.server.services.registry.RegistryViewForProviders
import org.apache.slider.providers.ProviderCompleted
import org.apache.hadoop.service.Service.STATE
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders
@@ -208,7 +206,6 @@ class MockProviderService implements ProviderService {
@Override
void bind(
StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
QueueAccess queueAccess,
List<Container> liveContainers) {
@@ -248,7 +245,6 @@ class MockProviderService implements ProviderService {
URL amWebURI,
URL agentOpsURI,
URL agentStatusURI,
- ServiceInstanceData registryInstanceData,
ServiceRecord serviceRecord) throws IOException {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
index 1c98e94..c49abec 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
@@ -52,7 +52,12 @@ public class TestClusterSpecificationBlock {
appState)
ProviderService providerService = new MockProviderService();
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
index 2bfd279..9292285 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.groovy
@@ -59,7 +59,12 @@ public class TestContainerStatsBlock extends BaseMockAppStateTest {
"undefined",
appState)
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index 86d3ef4..a3a4118 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.api.SliderClusterProtocol
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.model.mock.*
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
@@ -52,7 +51,12 @@ public class TestIndexBlock extends BaseMockAppStateTest {
"undefined",
appState)
- WebAppApiImpl inst = new WebAppApiImpl(clusterProto, providerAppState, providerService, null);
+ WebAppApiImpl inst = new WebAppApiImpl(
+ clusterProto,
+ providerAppState,
+ providerService,
+ null,
+ null);
Injector injector = Guice.createInjector(new AbstractModule() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 7e8fec9..7621a4d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.RoleKeys
@@ -689,19 +690,15 @@ class SliderTestUtils extends Assert {
}
public static void dumpRegistryInstances(
- List<CuratorServiceInstance<ServiceInstanceData>> instances) {
+ Map<String, ServiceRecord> instances) {
describe "service registry slider instances"
- JsonSerDeser<ServiceInstanceData> serDeser = new JsonSerDeser<>(
- ServiceInstanceData)
-
- instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
- ServiceInstanceData payload = svc.payload
- def json = serDeser.toJson(payload)
- log.info("service $svc payload=\n$json")
+ instances.each { Map.Entry<String, ServiceRecord> it ->
+ log.info(" $it.key : $it.value")
}
describe "end list service registry slider instances"
}
+
public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
describe "service registry instance IDs"
dumpCollection(instanceIds)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index c825fb8..452b03b 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -54,7 +54,6 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import java.io.File;
-import java.io.IOException;
import java.net.URI;
//import java.nio.file.FileVisitResult;
//import java.nio.file.Files;
@@ -139,7 +138,7 @@ public class TestAMAgentWebServices {
appState);
slider = new WebAppApiImpl(new MockSliderClusterProtocol(), providerAppState,
- new MockProviderService(), null);
+ new MockProviderService(), null, null);
MapOperations compOperations = new MapOperations();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 91aa2b0..20889ac 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -191,7 +191,7 @@ public class TestAMManagementWebServices extends JerseyTest {
appState);
slider = new WebAppApiImpl(new MockSliderClusterProtocol(), providerAppState,
- new MockProviderService(), null);
+ new MockProviderService(), null, null);
bind(SliderJacksonJaxbJsonProvider.class);
bind(MockSliderAMWebServices.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 0b1d0fb..7d09ee6 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -20,8 +20,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,10 +41,9 @@ public class TestAgentProviderService extends AgentProviderService {
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers) {
- super.bind(stateAccessor, reg, queueAccess, liveContainers);
+ super.bind(stateAccessor, queueAccess, liveContainers);
Map<String,String> dummyProps = new HashMap<String, String>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 634ebe2..9c53d2c 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -20,11 +20,12 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.core.zk.ZKIntegration
@@ -32,8 +33,6 @@ import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
/**
@@ -77,28 +76,24 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
// look up the registry entries for HBase
describe "service registry names"
- SliderRegistryService registryService = client.registry
- def names = registryService.getServiceTypes();
+ Map<String, ServiceRecord> records = RegistryOperationUtils.listServiceRecords(
+ client.registryOperations,
+ RegistryOperationUtils.serviceclassPath(
+ RegistryOperationUtils.homePathForCurrentUser(),
+ HBaseKeys.HBASE_SERVICE_TYPE
+ )
+ )
+
+ def names = records.keySet()
dumpRegistryServiceTypes(names)
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- client.listRegistryInstances();
- def hbaseInstances = registryService.findInstances( HBaseKeys.HBASE_SERVICE_TYPE, null)
+
+ def hbaseInstances = records.values()
assert hbaseInstances.size() == 1
- def hbaseService = hbaseInstances[0]
- assert hbaseService
- def hbaseServiceData = hbaseService.payload
+ ServiceRecord hbaseServiceData = hbaseInstances[0]
log.info "HBase service 0 == $hbaseServiceData"
assert hbaseServiceData.id
- assert hbaseServiceData.serviceType == HBaseKeys.HBASE_SERVICE_TYPE
-
- hbaseInstances = registryService.findInstances(
- HBaseKeys.HBASE_SERVICE_TYPE,
- clustername)
- assert hbaseInstances.size() == 1
- def hbaseServiceData2 = hbaseInstances[0].payload
- assert hbaseServiceData == hbaseServiceData2
RegistryRetriever retriever = new RegistryRetriever(hbaseServiceData)
log.info retriever.toString()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index f7bb8d2..a415524 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -20,14 +20,16 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
+import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
@CompileStatic
@@ -83,10 +85,15 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
// registry instances def names = client.listRegistryNames(clustername)
describe "service registry names"
- SliderRegistryService registry = cluster2Client.registry
- def names = registry.getServiceTypes();
- dumpRegistryServiceTypes(names)
+ RegistryOperations registry = cluster2Client.registryOperations
+ def home = RegistryOperationUtils.homePathForCurrentUser()
+ def names = RegistryOperationUtils.listServiceRecords(registry,
+ RegistryPathUtils.join(home, RegistryConstants.PATH_USER_SERVICES))
+ def stats = registry.listFull(
+ RegistryPathUtils.join(home, RegistryConstants.PATH_USER_SERVICES))
+
+ dumpCollection(stats)
List<String> instanceIds = sliderClient.listRegisteredSliderInstances()
@@ -95,18 +102,20 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
assert instanceIds.size() == 2
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ Map<String, ServiceRecord> instances =
sliderClient.listRegistryInstances()
dumpRegistryInstances(instances)
assert instances.size() == 2
- def hbaseInstances = registry.findInstances(
- HBaseKeys.HBASE_SERVICE_TYPE, null)
+ Map<String, ServiceRecord> hbaseInstances =
+ RegistryOperationUtils.listServiceRecords(registry,
+ RegistryOperationUtils.serviceclassPath(
+ RegistryOperationUtils.currentUser(),
+ HBaseKeys.HBASE_SERVICE_TYPE));
+
assert hbaseInstances.size() == 2
- def hbase1ServiceData = registry.findInstance(
- HBaseKeys.HBASE_SERVICE_TYPE, clustername1).payload
- def hbase2ServiceData = registry.findInstance(
- HBaseKeys.HBASE_SERVICE_TYPE, clustername2).payload
+ def hbase1ServiceData = hbaseInstances[clustername1]
+ def hbase2ServiceData = hbaseInstances[clustername2]
assert !(hbase1ServiceData == hbase2ServiceData)
clusterActionFreeze(cluster2Client, clustername2, "stop cluster 2")
[08/50] [abbrv] git commit: SLIDER-149 in sync with YARN-913 API
Changes
Posted by st...@apache.org.
SLIDER-149 in sync with YARN-913 API Changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e9380c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e9380c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e9380c0
Branch: refs/heads/develop
Commit: 6e9380c084a846eebf0c639d64626014e34db007
Parents: edf7286
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 11 17:24:14 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 11 17:24:14 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/SliderAppMaster.java | 2 +-
.../server/services/yarnregistry/YarnRegistryViewForProviders.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e9380c0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index f924ef5..193da3b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1009,7 +1009,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceRecord container = new ServiceRecord(
cid,
description,
- PersistencePolicies.CONTAINER);
+ PersistencePolicies.CONTAINER, null);
try {
yarnRegistryOperations.putComponent(cid, container);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e9380c0/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index bea7f78..fc7ec2a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -21,7 +21,7 @@ package org.apache.slider.server.services.yarnregistry;
import org.apache.hadoop.yarn.registry.client.binding.BindingUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
-import org.apache.hadoop.yarn.registry.client.types.CreateFlags;
+import org.apache.hadoop.yarn.registry.client.api.CreateFlags;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import java.io.IOException;
[41/50] [abbrv] git commit: minor cleanup of utility classes,
mostly formatting
Posted by st...@apache.org.
minor cleanup of utility classes, mostly formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c09205f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c09205f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c09205f6
Branch: refs/heads/develop
Commit: c09205f66725438854456379664fae8a0fa298ab
Parents: 7f7bb5b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 15:41:14 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 15:41:44 2014 -0700
----------------------------------------------------------------------
.../slider/common/tools/CoreFileSystem.java | 31 +++++++-------------
.../apache/slider/common/tools/SliderUtils.java | 14 ++++-----
2 files changed, 17 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index b6e6ecf..2ea371b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -158,11 +158,10 @@ public class CoreFileSystem {
* @throws java.io.IOException trouble
* @throws SliderException slider-specific exceptions
*/
- public Path createClusterDirectories(String clustername, Configuration conf) throws
- IOException,
- SliderException {
-
-
+ public Path createClusterDirectories(String clustername, Configuration conf)
+ throws IOException, SliderException {
+
+
Path clusterDirectory = buildClusterDirPath(clustername);
InstancePaths instancePaths = new InstancePaths(clusterDirectory);
createClusterDirectories(instancePaths);
@@ -204,8 +203,8 @@ public class CoreFileSystem {
*
* @param dir directory
* @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws org.apache.slider.core.exceptions.BadClusterStateException any cluster state problem
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
*/
public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
IOException,
@@ -572,7 +571,7 @@ public class CoreFileSystem {
*
* @param clustername name of the cluster
* @return the path to the spec.
- * @throws IOException IO problems
+ * @throws IOException IO problems
* @throws SliderException if the path isn't there
*/
public Path locateInstanceDefinition(String clustername) throws IOException,
@@ -591,23 +590,15 @@ public class CoreFileSystem {
* @throws IOException IO problems
* @throws SliderException if the cluster specification is not present
*/
- public void verifyClusterSpecExists(String clustername,
- Path clusterSpecPath) throws
- IOException,
+ public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
+ throws IOException,
SliderException {
if (!fileSystem.isFile(clusterSpecPath)) {
log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(clustername
- +
- "\n (definition not found at "
- +
- clusterSpecPath);
+ throw UnknownApplicationInstanceException.unknownInstance(
+ clustername + "\n (definition not found at " + clusterSpecPath);
}
}
-
- public Path fileToPath(File file) {
- return new Path(file.getAbsoluteFile().toURI());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 5313ab9..9c93753 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -511,8 +511,7 @@ public final class SliderUtils {
int length = separator.length();
String s = b.toString();
return (trailing || s.isEmpty()) ?
- s
- : (b.substring(0, b.length() - length));
+ s : (b.substring(0, b.length() - length));
}
/**
@@ -1589,7 +1588,7 @@ public final class SliderUtils {
}
- protected static void verifyIsFile(String program, File exe) throws
+ public static void verifyIsFile(String program, File exe) throws
FileNotFoundException {
if (!exe.isFile()) {
throw new FileNotFoundException(program
@@ -1599,7 +1598,7 @@ public final class SliderUtils {
}
}
- protected static void verifyFileSize(String program,
+ public static void verifyFileSize(String program,
File exe,
int minFileSize) throws FileNotFoundException {
if (exe.length() < minFileSize) {
@@ -1740,7 +1739,6 @@ public final class SliderUtils {
} catch (InterruptedException e) {
throw new InterruptedIOException(e.toString());
} catch (TimeoutException e) {
- log.debug("");
errorText = e.toString();
}
// error text: non null ==> operation failed
@@ -1843,11 +1841,11 @@ public final class SliderUtils {
/**
* validate if a file on HDFS can be open
*
- * @throws IOException
- * : the file can't be found or open
+ * @throws IOException the file can't be found or opened
* @throws URISyntaxException
*/
- public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException{
+ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr)
+ throws IOException, URISyntaxException{
URI pathURI = new URI(pathStr);
InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(pathURI));
if(inputStream == null){
[30/50] [abbrv] git commit: SLIDE 149;
sync with YARN-913 which has moved to specific exception for
NoPathPermissions (as hadoop.fs one didn't let me set text;
propagate through the codebase)
Posted by st...@apache.org.
SLIDE 149; sync with YARN-913 which has moved to specific exception for NoPathPermissions (as hadoop.fs one didn't let me set text; propagate through the codebase)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/969b734f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/969b734f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/969b734f
Branch: refs/heads/develop
Commit: 969b734fc03ac5684cbbcd326d475f44810560c0
Parents: a3e5b37
Author: Steve Loughran <st...@apache.org>
Authored: Sun Sep 28 11:22:47 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Sep 28 11:22:47 2014 +0100
----------------------------------------------------------------------
.../appmaster/web/rest/registry/PathEntryResource.java | 8 +-------
.../appmaster/web/rest/registry/RegistryResource.java | 12 +++---------
.../web/rest/registry/TestRegistryRestResources.groovy | 2 +-
3 files changed, 5 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/969b734f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
index 8790871..91da408 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
@@ -18,16 +18,10 @@
package org.apache.slider.server.appmaster.web.rest.registry;
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.util.List;
/**
@@ -38,7 +32,7 @@ import java.util.List;
public class PathEntryResource {
/**
- * Child nodes: as the full path to each element
+ * Child nodes: as the short path to each element
*/
public List<String> nodes;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/969b734f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
index 23f7261..fc41e7d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -19,22 +19,19 @@
package org.apache.slider.server.appmaster.web.rest.registry;
import com.google.inject.Singleton;
-import org.apache.hadoop.fs.PathAccessDeniedException;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException;
import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.exceptions.NoPathPermissionsException;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
@@ -42,13 +39,11 @@ import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
-import java.util.List;
/**
- * This is the read-only view of the slider YARN registry
+ * This is the read-only view of the YARN registry.
*
* Model:
* <ol>
@@ -126,7 +121,7 @@ public class RegistryResource {
throw new NotFoundException("Not found: " + path);
} catch (AuthenticationFailedException e) {
throw new ForbiddenException(path);
- } catch (PathAccessDeniedException e) {
+ } catch (NoPathPermissionsException e) {
throw new ForbiddenException(path);
} catch (Exception e) {
log.error("Error during generation of response: {}", e, e);
@@ -139,7 +134,6 @@ public class RegistryResource {
* Build from the registry, filling up the children and service records.
* If there is no service record at the end of the path, that entry is
* null
- * @param registry registry operations
* @param path path to query
* @return the built up record
* @throws IOException problems
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/969b734f/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
index 877ba56..9dd3092 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -125,7 +125,7 @@ class TestRegistryRestResources extends AgentTestBase {
def responseStr = processResponse(userServicesURL, response)
assert responseStr.contains("\"nodes\"")
- assert responseStr.contains("/${SliderKeys.APP_TYPE}")
+ assert responseStr.contains(SliderKeys.APP_TYPE)
entryResource = jsonBuilder.get(PathEntryResource.class)
assert entryResource.nodes.size() == 1;
[46/50] [abbrv] git commit: SLIDER-249 test doesn't compile without
"YARN-2571 RM to support YARN registry" extension of YARN-913;
rm'd an assertion
Posted by st...@apache.org.
SLIDER-249 test doesn't compile without "YARN-2571 RM to support YARN registry" extension of YARN-913; rm'd an assertion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3a759d1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3a759d1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3a759d1e
Branch: refs/heads/develop
Commit: 3a759d1ec111fbc57cbe0697422052a049dca031
Parents: 02623b1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 11:47:23 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 11:47:23 2014 -0700
----------------------------------------------------------------------
.../slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3a759d1e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 97e995d..83774d2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -76,10 +76,6 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def registryOperations = microZKCluster.registryOperations
registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
- // verify the cluster has the YARN reg service live
- def rmRegistryService = miniCluster.getResourceManager(0).RMContext.registry
- assert rmRegistryService
-
ServiceLauncher<SliderClient> launcher
launcher = createStandaloneAM(clustername, true, false)
SliderClient client = launcher.service
[43/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9175aa7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9175aa7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9175aa7e
Branch: refs/heads/develop
Commit: 9175aa7e5acd704faf9075076f5b1be61b3bb70f
Parents: 032dacf 6b19cb8
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 09:55:26 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 09:55:26 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 9 +++++----
app-packages/accumulo/pom.xml | 8 ++++++--
2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8c7f6b23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8c7f6b23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8c7f6b23
Branch: refs/heads/develop
Commit: 8c7f6b23d5cf0202718b38d182ff5624f4bd8955
Parents: 8b2d4d5 94df22c
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 24 15:00:51 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 24 15:00:51 2014 +0100
----------------------------------------------------------------------
----------------------------------------------------------------------
[04/50] [abbrv] git commit: SLIDER-149 in sync with YARN-913-patch-002
Posted by st...@apache.org.
SLIDER-149 in sync with YARN-913-patch-002
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5784c4ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5784c4ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5784c4ab
Branch: refs/heads/develop
Commit: 5784c4ab5e2cae45625d2dfa745802c69d5f19ab
Parents: 901a673
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 9 16:12:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 9 16:12:46 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 7 +-
.../server/appmaster/SliderAppMaster.java | 18 +-
.../TestStandaloneYarnRegistryAM.groovy | 378 ++++++++++++++++++
.../agent/standalone/TestYarnRegistryAM.groovy | 379 -------------------
4 files changed, 392 insertions(+), 390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8219fd7..ed3bc0e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -136,6 +136,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -2376,13 +2377,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public List<ServiceRecord> actionRegistryListYarn(
+ public Collection<ServiceRecord> actionRegistryListYarn(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
RegistryOperationsService operations = getRegistryOperations();
- List<ServiceRecord> serviceRecords;
+ Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
String serviceclassPath =
BindingUtils.serviceclassPath(BindingUtils.currentUser(),
@@ -2394,7 +2395,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ serviceclassPath);
}
serviceRecords =
- RecordOperations.extractServiceRecords(operations, listDir);
+ RecordOperations.extractServiceRecords(operations, listDir).values();
} else {
ServiceRecord instance = lookupServiceRecord(registryArgs);
serviceRecords = new ArrayList<ServiceRecord>(1);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9615b6f..4b5e0b3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -52,11 +52,12 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.server.services.ResourceManagerRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -929,6 +930,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceRecord serviceRecord = new ServiceRecord();
String serviceID = appid.toString();
serviceRecord.id = serviceID;
+ serviceRecord.persistence = PersistencePolicies.APPLICATION;
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -967,7 +969,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and an ephemeral binding to the app
yarnRegistryOperations.putComponent(
- RegistryTypeUtils.yarnIdToDnsId(appAttemptID.toString()),
+ RegistryPathUtils.encodeYarnID(appAttemptID.toString()),
serviceRecord,
false);
@@ -982,10 +984,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
protected void setupInitialRegistryPaths() throws IOException {
- if (registryOperations instanceof ResourceManagerRegistryService) {
- ResourceManagerRegistryService rmRegOperations =
- (ResourceManagerRegistryService) registryOperations;
- rmRegOperations.createUserPath(service_user_name);
+ if (registryOperations instanceof RMRegistryOperationsService) {
+ RMRegistryOperationsService rmRegOperations =
+ (RMRegistryOperationsService) registryOperations;
+ rmRegOperations.createHomeDirectory(service_user_name);
}
}
@@ -1003,7 +1005,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
// this is where component registrations will go
log.info("Registering component {}", id);
- String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
ServiceRecord container = new ServiceRecord(
cid,
description,
@@ -1027,7 +1029,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
public void unregisterComponent(ContainerId id) {
log.info("Unregistering component {}", id);
- String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
try {
yarnRegistryOperations.rmComponent(cid);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
new file mode 100644
index 0000000..c7291a8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+
+import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
+import org.apache.slider.core.registry.docstore.UriMap
+import org.apache.slider.core.registry.info.CustomRegistryConstants
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Test
+
+/**
+ * work with a YARN registry
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
+
+
+ public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
+
+ @Test
+ public void testYarnRegistryAM() throws Throwable {
+
+
+ describe "create a masterless AM then perform YARN registry operations on it"
+
+
+ String clustername = createMiniCluster(configuration, 1, true)
+
+ // get local binding
+ def registryOperations = microZKCluster.registryOperations
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+
+ // verify the cluster has the YARN reg service live
+ def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
+ assert rmRegistryService
+
+ ServiceLauncher<SliderClient> launcher
+ launcher = createStandaloneAM(clustername, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)clusterNodes).size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def yarnRegistryClient = client.yarnAppListClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ // sleep to allow registration to complete
+ sleep(5000)
+
+
+
+
+ try {
+ def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ log.info("yarn service registry: \n${yarnRegistryDump}\n")
+ } catch (IOException ignored) {
+
+ }
+
+
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ def self = currentUser()
+ RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
+ dumpArray(serviceTypes)
+
+ def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
+
+ Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
+ registryService,
+ registryService.listDir(recordsPath))
+ def serviceRecords = recordMap.values();
+ dumpCollection(serviceRecords)
+ assert serviceRecords.size() == 1
+
+ def serviceInstance = serviceRecords[0]
+ log.info(serviceInstance.toString())
+
+ assert 2 <= serviceInstance.external.size()
+
+ // hit the registry web page
+
+ def registryEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+ describe("Registry WADL @ $registryURL")
+
+ def publisherEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API)
+
+ def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
+ def setlisting = GET(publisherURL)
+
+ log.info(setlisting)
+
+ UriMap uris = uriMapDeser.fromJson(setlisting)
+ assert uris.uris[RestPaths.SLIDER_CONFIGSET]
+ def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
+
+ assert publishedYarnSite.empty
+
+ //get the full URL
+ def yarnSitePublisher = appendToURL(publisher,
+ RestPaths.SLIDER_CONFIGSET,
+ ARTIFACT_NAME)
+
+ String confJSON = GET(yarnSitePublisher)
+// log.info(confJSON)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+
+ publishedYarnSite = confSerDeser.fromJson(confJSON)
+
+ assert !publishedYarnSite.empty
+
+
+ //get the XML
+ def yarnSiteXML = yarnSitePublisher + ".xml"
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String properties = GET(yarnSitePublisher + ".properties")
+ Properties parsedProps = new Properties()
+ parsedProps.load(new StringReader(properties))
+ assert parsedProps.size() > 0
+ def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
+ def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
+ assert rmAddrFromDownloadedProperties
+ assert rmHostnameFromDownloadedProperties
+
+ String json = GET(yarnSitePublisher + ".json")
+
+
+
+ describe("Registry List")
+ log.info(GET(registryURL))
+
+
+ describe "Registry Retrieval Class"
+ // retrieval
+
+ RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[ARTIFACT_NAME]
+
+
+ describe "verify SLIDER-52 processing"
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ ARTIFACT_NAME,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
+ def rmHostnameViaClientSideXML = parsedProps.get(
+ YarnConfiguration.RM_HOSTNAME)
+ assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
+ def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
+ describe "fetch missing artifact"
+ try {
+ retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ try {
+ retriever.getConfigurations(false)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.verbose = true
+
+ // list all
+ registryArgs.list = true;
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+
+ // list a named instance and expect a failure
+ registryArgs.list = true;
+ registryArgs.name = "unknown"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ // list all instances of an alternate type and expect failure
+ registryArgs.list = true;
+ registryArgs.name = null
+ registryArgs.serviceType = "org-apache-hadoop"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ registryArgs.serviceType = ""
+
+ //set the name
+ registryArgs.name = clustername;
+ registryArgs.serviceType = SliderKeys.APP_TYPE
+
+
+ //now expect list to work
+ describe registryArgs.toString()
+
+ def listedInstance = client.actionRegistryListYarn(registryArgs)
+ assert listedInstance[0].id == serviceInstance.id
+
+
+ // listconf
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+ // listconf --internal
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ registryArgs.internal = true
+ describe registryArgs.toString()
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = false
+ registryArgs.internal = false
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registryArgs.getConf = yarn_site_config
+
+ //properties format
+ registryArgs.format = "properties"
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+
+ File outputDir = new File("target/test_standalone_registry_am/output")
+ outputDir.mkdirs()
+
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(clustername)
+ registryArgs.getConf = yarn_site_config
+ registryArgs.dest = outputDir
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".xml").exists()
+
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
+ describe registryArgs.toString()
+
+ def unknownFilename = "undefined-file"
+ registryArgs.getConf = unknownFilename
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ describe "stop cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = yarnRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
deleted file mode 100644
index bd15bb4..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.PathNotFoundException
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
-
-import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.api.ClusterNode
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.ActionRegistryArgs
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.docstore.PublishedConfiguration
-import org.apache.slider.core.registry.docstore.UriMap
-import org.apache.slider.core.registry.info.CustomRegistryConstants
-import org.apache.slider.core.registry.retrieve.RegistryRetriever
-import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.junit.Test
-
-/**
- * work with a YARN registry
- */
-@CompileStatic
-@Slf4j
-
-class TestYarnRegistryAM extends AgentMiniClusterTestBase {
-
-
- public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
-
- @Test
- public void testYarnRegistryAM() throws Throwable {
-
-
- describe "create a masterless AM then perform YARN registry operations on it"
-
-
- String clustername = createMiniCluster(configuration, 1, true)
-
- // get local binding
- def registryOperations = microZKCluster.registryOperations
- registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
-
- // verify the cluster has the YARN reg service live
- def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
- assert rmRegistryService
-
-
-
-
-
- ServiceLauncher<SliderClient> launcher
- launcher = createStandaloneAM(clustername, true, false)
- SliderClient client = launcher.service
- addToTeardown(client);
-
- ApplicationReport report = waitForClusterLive(client)
- logReport(report)
- List<ApplicationReport> apps = client.applications;
-
- List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
- SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)clusterNodes).size() == 1
-
- ClusterNode masterNode = clusterNodes[0]
- log.info("Master node = ${masterNode}");
-
- List<ClusterNode> nodes
- String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
- assert uuids.length == 1;
- nodes = client.listClusterNodes(uuids);
- assert ((List<ClusterNode>)nodes).size() == 1;
- describe "AM Node UUID=${uuids[0]}"
-
- nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)nodes).size() == 1;
- nodes = listNodesInRole(client, "")
- assert ((List<ClusterNode>)nodes).size() == 1;
- ClusterNode master = nodes[0]
- assert master.role == SliderKeys.COMPONENT_AM
-
-
-
-
- String username = client.username
- def yarnRegistryClient = client.yarnAppListClient
- describe("list of all applications")
- logApplications(apps)
- describe("apps of user $username")
- List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
- logApplications(userInstances)
- assert userInstances.size() == 1
- describe("named app $clustername")
- ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
- logReport(instance)
- assert instance != null
-
- // sleep to allow registration to complete
- sleep(5000)
-
-
-
-
- try {
- def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
- log.info("yarn service registry: \n${yarnRegistryDump}\n")
- } catch (IOException ignored) {
-
- }
-
-
- describe "service registry names"
- def registryService = client.registryOperations
-
- def self = currentUser()
- RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
- dumpArray(serviceTypes)
-
- def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
-
- def serviceRecords = RecordOperations.extractServiceRecords(registryService,
- registryService.listDir(recordsPath))
- dumpCollection(serviceRecords)
- assert serviceRecords.size() == 1
-
- def serviceInstance = serviceRecords[0]
- log.info(serviceInstance.toString())
-
- assert 2 <= serviceInstance.external.size()
-
- // hit the registry web page
-
- def registryEndpoint = serviceInstance.getExternalEndpoint(
- CustomRegistryConstants.REGISTRY_REST_API)
- assert registryEndpoint != null
- def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
- describe("Registry WADL @ $registryURL")
-
- def publisherEndpoint = serviceInstance.getExternalEndpoint(
- CustomRegistryConstants.PUBLISHER_REST_API)
-
- def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
- def publisher = publisherURL.toString()
- describe("Publisher")
-
- JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
- def setlisting = GET(publisherURL)
-
- log.info(setlisting)
-
- UriMap uris = uriMapDeser.fromJson(setlisting)
- assert uris.uris[RestPaths.SLIDER_CONFIGSET]
- def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
- JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
- PublishedConfigSet)
- def configSet = serDeser.fromJson(publishedJSON)
- assert configSet.size() >= 1
- assert configSet.contains(ARTIFACT_NAME)
- PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
-
- assert publishedYarnSite.empty
-
- //get the full URL
- def yarnSitePublisher = appendToURL(publisher,
- RestPaths.SLIDER_CONFIGSET,
- ARTIFACT_NAME)
-
- String confJSON = GET(yarnSitePublisher)
-// log.info(confJSON)
- JsonSerDeser< PublishedConfiguration> confSerDeser =
- new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
-
- publishedYarnSite = confSerDeser.fromJson(confJSON)
-
- assert !publishedYarnSite.empty
-
-
- //get the XML
- def yarnSiteXML = yarnSitePublisher + ".xml"
-
-
- String confXML = GET(yarnSiteXML)
- log.info("Conf XML at $yarnSiteXML = \n $confXML")
-
- String properties = GET(yarnSitePublisher + ".properties")
- Properties parsedProps = new Properties()
- parsedProps.load(new StringReader(properties))
- assert parsedProps.size() > 0
- def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
- def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
- assert rmAddrFromDownloadedProperties
- assert rmHostnameFromDownloadedProperties
-
- String json = GET(yarnSitePublisher + ".json")
-
-
-
- describe("Registry List")
- log.info(GET(registryURL))
-
-
- describe "Registry Retrieval Class"
- // retrieval
-
- RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
- log.info retriever.toString()
-
- assert retriever.hasConfigurations(true)
- PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
- dumpConfigurationSet(externalConfSet)
- assert externalConfSet[ARTIFACT_NAME]
-
-
- describe "verify SLIDER-52 processing"
- def yarnSite = retriever.retrieveConfiguration(
- externalConfSet,
- ARTIFACT_NAME,
- true)
- assert !yarnSite.empty
- def siteXML = yarnSite.asConfiguration()
- def rmHostnameViaClientSideXML = parsedProps.get(
- YarnConfiguration.RM_HOSTNAME)
- assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
- def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
-
- log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
- assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
-
- describe "fetch missing artifact"
- try {
- retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
- describe "Internal configurations"
- assert !retriever.hasConfigurations(false)
- try {
- retriever.getConfigurations(false)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
-
-
- // retrieval via API
- ActionRegistryArgs registryArgs = new ActionRegistryArgs()
- registryArgs.verbose = true
-
- // list all
- registryArgs.list = true;
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
-
- // list a named instance and expect a failure
- registryArgs.list = true;
- registryArgs.name = "unknown"
- try {
- client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
- // expected
- }
-
- // list all instances of an alternate type and expect failure
- registryArgs.list = true;
- registryArgs.name = null
- registryArgs.serviceType = "org-apache-hadoop"
- try {
- client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
- // expected
- }
-
- registryArgs.serviceType = ""
-
- //set the name
- registryArgs.name = clustername;
- registryArgs.serviceType = SliderKeys.APP_TYPE
-
-
- //now expect list to work
- describe registryArgs.toString()
-
- def listedInstance = client.actionRegistryListYarn(registryArgs)
- assert listedInstance[0].id == serviceInstance.id
-
-
- // listconf
- registryArgs.list = false;
- registryArgs.listConf = true
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
- // listconf --internal
- registryArgs.list = false;
- registryArgs.listConf = true
- registryArgs.internal = true
- describe registryArgs.toString()
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- registryArgs.list = false;
- registryArgs.listConf = false
- registryArgs.internal = false
-
- def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
- registryArgs.getConf = yarn_site_config
-
- //properties format
- registryArgs.format = "properties"
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
-
- File outputDir = new File("target/test_standalone_registry_am/output")
- outputDir.mkdirs()
-
- // create a new registry args with the defaults back in
- registryArgs = new ActionRegistryArgs(clustername)
- registryArgs.getConf = yarn_site_config
- registryArgs.dest = outputDir
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".xml").exists()
-
- registryArgs.format = "properties"
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".properties").exists()
-
- describe registryArgs.toString()
-
- def unknownFilename = "undefined-file"
- registryArgs.getConf = unknownFilename
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- describe "stop cluster"
- //now kill that cluster
- assert 0 == clusterActionFreeze(client, clustername)
- //list it & See if it is still there
- ApplicationReport oldInstance = yarnRegistryClient.findInstance(
- clustername)
- assert oldInstance != null
- assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
-
-
-
- }
-}
[35/50] [abbrv] git commit: SLIDER-365 slider "resolve" command to
retrieve service record
Posted by st...@apache.org.
SLIDER-365 slider "resolve" command to retrieve service record
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d084e9e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d084e9e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d084e9e8
Branch: refs/heads/develop
Commit: d084e9e86f145655af3eafe2da76db08269db54a
Parents: c51f2f0
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 6 13:53:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 6 13:53:49 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 56 ++++++++++--
.../common/params/ActionRegistryArgs.java | 9 +-
.../slider/common/params/ActionResolveArgs.java | 92 ++++++++++++++++++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 65 ++++++++------
.../slider/common/params/SliderActions.java | 1 +
6 files changed, 183 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 26e5707..a5bbccc 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -72,6 +72,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
@@ -332,9 +333,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
+ } else if (ACTION_DIAGNOSTIC.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
} else if (ACTION_GETCONF.equals(action)) {
@@ -351,8 +354,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (ACTION_LIST.equals(action)) {
exitCode = actionList(clusterName);
} else if (ACTION_REGISTRY.equals(action)) {
- exitCode = actionRegistry(
- serviceArgs.getActionRegistryArgs());
+ exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
+ } else if (ACTION_RESOLVE.equals(action)) {
+ exitCode = actionResolve(serviceArgs.getActionResolveArgs());
} else if (ACTION_STATUS.equals(action)) {
exitCode = actionStatus(clusterName,
serviceArgs.getActionStatusArgs());
@@ -362,8 +366,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (ACTION_VERSION.equals(action)) {
exitCode = actionVersion();
- } else if (ACTION_DIAGNOSTIC.equals(action)) {
- exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
"Unimplemented: " + action);
@@ -2387,13 +2389,51 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Registry operation
*
+ * @param args registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ public int actionResolve(ActionResolveArgs args) throws
+ YarnException,
+ IOException {
+ // as this is also a test entry point, validate
+ // the arguments
+ args.validate();
+ RegistryOperations operations = getRegistryOperations();
+ String serviceclassPath = args.path;
+ Collection<ServiceRecord> serviceRecords;
+ try {
+ if (args.list) {
+ actionRegistryListYarn(args);
+ } else {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+ serviceRecords = new ArrayList<ServiceRecord>(1);
+ serviceRecords.add(instance);
+ }
+// JDK7
+ } catch (FileNotFoundException e) {
+ log.info("{}", e);
+ log.debug("{}", e, e);
+ return EXIT_NOT_FOUND;
+ } catch (PathNotFoundException e) {
+ log.info("{}", e);
+ log.debug("{}", e, e);
+ return EXIT_NOT_FOUND;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ /**
+ * Registry operation
+ *
* @param registryArgs registry Arguments
* @return 0 for success, -1 for some issues that aren't errors, just failures
* to retrieve information (e.g. no configurations for that entry)
* @throws YarnException YARN problems
* @throws IOException Network or other problems
*/
- @VisibleForTesting
public int actionRegistry(ActionRegistryArgs registryArgs) throws
YarnException,
IOException {
@@ -2418,11 +2458,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
// JDK7
} catch (FileNotFoundException e) {
- log.info("{}", e.toString());
+ log.info("{}", e);
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e.toString());
+ log.info("{}", e);
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index b3d2f9c..6de9a45 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -42,16 +42,17 @@ import java.io.File;
public class ActionRegistryArgs extends AbstractActionArgs {
-
public static final String USAGE =
"Usage: " + SliderActions.ACTION_REGISTRY
- + " <"
+ + " ("
+ Arguments.ARG_LIST + "|"
+ Arguments.ARG_LISTCONF + "|"
+ Arguments.ARG_LISTFILES + "|"
+ Arguments.ARG_GETCONF + "> "
+ Arguments.ARG_NAME + " <name> "
+ + " )"
+ "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
;
@@ -67,7 +68,6 @@ public class ActionRegistryArgs extends AbstractActionArgs {
return ACTION_REGISTRY;
}
-
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
@@ -89,7 +89,6 @@ public class ActionRegistryArgs extends AbstractActionArgs {
description = "get configuration")
public String getConf;
-
@Parameter(names = {ARG_LISTFILES},
description = "list files")
public String listFiles;
@@ -103,7 +102,7 @@ public class ActionRegistryArgs extends AbstractActionArgs {
description = "Format for a response: <xml|json|properties>")
public String format = ConfigFormat.XML.toString() ;
- @Parameter(names = {ARG_DEST},
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
description = "Output destination")
public File dest;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
new file mode 100644
index 0000000..9b6a5f1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
+import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
+import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+
+/**
+ * Resolve registry entries
+ *
+ * --path {path}
+ * --out {destfile}
+ * --verbose
+ * --list
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+ commandDescription = DESCRIBE_ACTION_REGISTRY)
+public class ActionResolveArgs extends AbstractActionArgs {
+
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_RESOLVE
+ + " "
+ + Arguments.ARG_PATH + " <path> "
+ + "[" + Arguments.ARG_LIST + "] "
+ + "[" + Arguments.ARG_VERBOSE + "] "
+ + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ ;
+ public ActionResolveArgs() {
+ }
+
+ @Override
+ public String getActionName() {
+ return ACTION_RESOLVE;
+ }
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_LIST},
+ description = "list services")
+ public boolean list;
+
+
+ @Parameter(names = {ARG_PATH},
+ description = "resolve a path")
+ public String path;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output destination")
+ public File out;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "verbose output")
+ public boolean verbose;
+
+ @Parameter(names = {ARG_INTERNAL},
+ description = "fetch internal registry entries")
+ public boolean internal;
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 2b45ce8..bbd7da6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -62,6 +62,7 @@ public interface Arguments {
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
String ARG_PACKAGE = "--package";
+ String ARG_PATH = "--path";
String ARG_PROVIDER = "--provider";
String ARG_RESOURCES = "--resources";
String ARG_RES_COMP_OPT = "--rescompopt";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index cd981b1..0fffffb 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -58,6 +58,7 @@ public class ClientArgs extends CommonArgs {
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
+ private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
@@ -77,25 +78,26 @@ public class ClientArgs extends CommonArgs {
protected void addActionArguments() {
addActions(
- actionAMSuicideArgs,
- actionBuildArgs,
- actionCreateArgs,
- actionUpdateArgs,
- actionDestroyArgs,
- actionExistsArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionGetConfArgs,
- actionKillContainerArgs,
- actionListArgs,
- actionRegistryArgs,
- actionStatusArgs,
- actionThawArgs,
- actionHelpArgs,
- actionVersionArgs,
- actionInstallPackageArgs,
- actionDiagnosticArgs
- );
+ actionAMSuicideArgs,
+ actionBuildArgs,
+ actionCreateArgs,
+ actionUpdateArgs,
+ actionDestroyArgs,
+ actionDiagnosticArgs,
+ actionExistsArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
+ actionGetConfArgs,
+ actionHelpArgs,
+ actionInstallPackageArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionRegistryArgs,
+ actionResolveArgs,
+ actionStatusArgs,
+ actionThawArgs,
+ actionVersionArgs
+ );
}
@Override
@@ -173,6 +175,10 @@ public class ClientArgs extends CommonArgs {
return actionRegistryArgs;
}
+ public ActionResolveArgs getActionResolveArgs() {
+ return actionResolveArgs;
+ }
+
public ActionStatusArgs getActionStatusArgs() {
return actionStatusArgs;
}
@@ -200,12 +206,6 @@ public class ClientArgs extends CommonArgs {
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
- } else if (SliderActions.ACTION_UPDATE.equals(action)) {
- bindCoreAction(actionUpdateArgs);
-
- } else if (SliderActions.ACTION_INSTALL_PACKAGE.equals(action)) {
- bindCoreAction(actionInstallPackageArgs);
-
}else if (SliderActions.ACTION_FREEZE.equals(action)) {
bindCoreAction(actionFreezeArgs);
@@ -218,6 +218,9 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_DESTROY.equals(action)) {
bindCoreAction(actionDestroyArgs);
+ } else if (SliderActions.ACTION_DIAGNOSTIC.equals(action)) {
+ bindCoreAction(actionDiagnosticArgs);
+
} else if (SliderActions.ACTION_EXISTS.equals(action)) {
bindCoreAction(actionExistsArgs);
@@ -231,6 +234,9 @@ public class ClientArgs extends CommonArgs {
SliderActions.ACTION_USAGE.equals(action)) {
bindCoreAction(actionHelpArgs);
+ } else if (SliderActions.ACTION_INSTALL_PACKAGE.equals(action)) {
+ bindCoreAction(actionInstallPackageArgs);
+
} else if (SliderActions.ACTION_KILL_CONTAINER.equals(action)) {
bindCoreAction(actionKillContainerArgs);
@@ -240,15 +246,18 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_REGISTRY.equals(action)) {
bindCoreAction(actionRegistryArgs);
+ } else if (SliderActions.ACTION_RESOLVE.equals(action)) {
+ bindCoreAction(actionRegistryArgs);
+
} else if (SliderActions.ACTION_STATUS.equals(action)) {
bindCoreAction(actionStatusArgs);
+ } else if (SliderActions.ACTION_UPDATE.equals(action)) {
+ bindCoreAction(actionUpdateArgs);
+
} else if (SliderActions.ACTION_VERSION.equals(action)) {
bindCoreAction(actionVersionArgs);
- } else if (SliderActions.ACTION_DIAGNOSTIC.equals(action)) {
- bindCoreAction(actionDiagnosticArgs);
-
} else if (action == null || action.isEmpty()) {
throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d084e9e8/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 29fd098..7400f46 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -40,6 +40,7 @@ public interface SliderActions {
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
+ String ACTION_RESOLVE = "resolve";
String ACTION_STATUS = "status";
String ACTION_THAW = "start";
String ACTION_USAGE = "usage";
[16/50] [abbrv] git commit: SLIDER-149/YARN-913 tracking changes
Posted by st...@apache.org.
SLIDER-149/YARN-913 tracking changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/821d3066
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/821d3066
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/821d3066
Branch: refs/heads/develop
Commit: 821d3066496c0b1ba581b0231b63f9bf5e5e478b
Parents: 2a28822
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:05:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:05:03 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/821d3066/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index d2b9295..1fd378b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -57,7 +57,7 @@ import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService
import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.server.services.RMRegistryOperationsService;
+import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -984,7 +984,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return new ResourceManagerRegistryService("YarnRegistry");
}
*/
-
+
+ /**
+ * TODO: purge this once RM is doing the work
+ * @throws IOException
+ */
protected void setupInitialRegistryPaths() throws IOException {
if (registryOperations instanceof RMRegistryOperationsService) {
RMRegistryOperationsService rmRegOperations =
[39/50] [abbrv] git commit: SLIDER-149/YARN-913 move registry package
to org.apache.hadoop.registry
Posted by st...@apache.org.
SLIDER-149/YARN-913 move registry package to org.apache.hadoop.registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfbc67af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfbc67af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfbc67af
Branch: refs/heads/develop
Commit: bfbc67affeb711bf6d20aa7ca76793ea840660b9
Parents: 29c9824
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 11:45:26 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 11:45:26 2014 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 6 +++---
.../org/apache/slider/client/SliderClient.java | 18 +++++++++---------
.../core/registry/retrieve/RegistryRetriever.java | 10 ++++------
.../slider/providers/AbstractProviderService.java | 10 +++++-----
.../apache/slider/providers/ProviderService.java | 2 +-
.../providers/agent/AgentProviderService.java | 7 +++----
.../slideram/SliderAMProviderService.java | 6 ++----
.../slider/server/appmaster/SliderAppMaster.java | 14 +++++++-------
.../server/appmaster/state/RoleInstance.java | 6 +++---
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../slider/server/appmaster/web/WebAppApi.java | 2 +-
.../server/appmaster/web/WebAppApiImpl.java | 2 +-
.../web/rest/registry/PathEntryResource.java | 2 +-
.../web/rest/registry/RegistryResource.java | 10 +++++-----
.../utility/AbstractSliderLaunchedService.java | 6 +++---
.../YarnRegistryViewForProviders.java | 10 +++++-----
.../TestStandaloneYarnRegistryAM.groovy | 14 +++++++-------
.../model/mock/MockProviderService.groovy | 2 +-
.../web/rest/registry/PathEntryMarshalling.groovy | 2 +-
.../registry/TestRegistryRestMarshalling.groovy | 2 +-
.../registry/TestRegistryRestResources.groovy | 6 +++---
.../org/apache/slider/test/MicroZKCluster.groovy | 6 +++---
.../org/apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnZKMiniClusterTestBase.groovy | 4 +---
.../providers/hbase/HBaseProviderService.java | 8 ++++----
.../hbase/minicluster/live/TestHBaseMaster.groovy | 6 +++---
.../minicluster/live/TestTwoLiveClusters.groovy | 8 ++++----
27 files changed, 83 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 2f6674e..bb9abba 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -25,7 +25,7 @@ import org.apache.hadoop.security.ProviderUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.alias.CredentialProvider
import org.apache.hadoop.security.alias.CredentialProviderFactory
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
@@ -39,8 +39,8 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.currentUser
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.servicePath
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index b3365a7..185e77d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -41,15 +41,15 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*;
-
-import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
+
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index df1d675..101efb2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -25,12 +25,10 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.yarn.registry.client.exceptions.RegistryIOException;
-import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 9af970c..04f6d8c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -23,11 +23,11 @@ import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
-import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index d78ca0f..b080ecb 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a3010ea..ca6ae71 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,9 +28,9 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -52,7 +52,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index e28f5a8..601c3f9 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -37,7 +37,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.agent.AgentKeys;
@@ -51,7 +50,6 @@ import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5f7ebe6..824951c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -54,13 +54,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index df5e47e..c6d8f4c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -22,9 +22,9 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.tools.SliderUtils;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 34080d6..7f30440 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -21,7 +21,7 @@ import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.webapp.Dispatcher;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 8d91fd4..b7e749e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -16,7 +16,7 @@
*/
package org.apache.slider.server.appmaster.web;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.AppState;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 52b53a9..4c0aa6a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -16,7 +16,7 @@
*/
package org.apache.slider.server.appmaster.web;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.ProviderRole;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
index 91da408..efb09a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
@@ -18,7 +18,7 @@
package org.apache.slider.server.appmaster.web.rest.registry;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
index fc41e7d..70c0826 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -20,11 +20,11 @@ package org.apache.slider.server.appmaster.web.rest.registry;
import com.google.inject.Singleton;
import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException;
-import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.yarn.registry.client.exceptions.NoPathPermissionsException;
-import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.slider.server.appmaster.web.WebAppApi;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index ada0047..acdb882 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -19,9 +19,9 @@
package org.apache.slider.server.services.utility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index 10ca85c..4eb91ab 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -19,12 +19,12 @@
package org.apache.slider.server.services.yarnregistry;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.registry.client.api.BindFlags;
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 62b39f0..1f65d2f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -23,15 +23,15 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
-import org.apache.hadoop.yarn.registry.client.impl.RegistryOperationsClient
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.registry.client.impl.RegistryOperationsClient
+import org.apache.hadoop.registry.client.types.RegistryPathStatus
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 07fa327..675aec5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -25,7 +25,7 @@ import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
index 3883b25..937bebf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
@@ -18,7 +18,7 @@
package org.apache.slider.server.appmaster.web.rest.registry
-import org.apache.hadoop.yarn.registry.client.binding.JsonSerDeser
+import org.apache.hadoop.registry.client.binding.JsonSerDeser
class PathEntryMarshalling
extends JsonSerDeser<PathEntryResource> {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
index d67dfeb..53dda35 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
@@ -19,7 +19,7 @@
package org.apache.slider.server.appmaster.web.rest.registry
import groovy.transform.CompileStatic
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
index ee678f9..43d109d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -24,9 +24,9 @@ import com.sun.jersey.api.client.UniformInterfaceException
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
index c2d6a32..3d1a2d7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
@@ -21,9 +21,9 @@ package org.apache.slider.test
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
-import org.apache.hadoop.yarn.registry.client.impl.zk.RegistryOperationsService
-import org.apache.hadoop.yarn.registry.server.services.MicroZookeeperService
+import org.apache.hadoop.registry.client.api.RegistryOperations
+import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService
+import org.apache.hadoop.registry.server.services.MicroZookeeperService
import org.apache.slider.common.tools.SliderUtils
@Slf4j
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index c5e41f7..0b45a5d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.RoleKeys
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index 197d94d..e182ed2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -23,15 +23,13 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.core.zk.BlockingZKWatcher
import org.apache.slider.core.zk.ZKIntegration
import java.util.concurrent.atomic.AtomicBoolean
-import static org.apache.slider.common.SliderKeys.*;
-import static org.apache.slider.common.SliderXMLConfKeysForTesting.*;
import static org.apache.slider.test.KeysForTests.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 6375d6f..a36ae70 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -21,10 +21,10 @@ package org.apache.slider.providers.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index b73432a..79193c3 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -20,9 +20,9 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
-import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbc67af/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index bb652b3..b7212d9 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -20,10 +20,10 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
-import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
-import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils
-import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.registry.client.api.RegistryOperations
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
[27/50] [abbrv] git commit: SLIDER-149;
in sync with registry code. HBase provider test working
Posted by st...@apache.org.
SLIDER-149; in sync with registry code. HBase provider test working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46fb5dbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46fb5dbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46fb5dbd
Branch: refs/heads/develop
Commit: 46fb5dbdb35c5b072771a394ec339f1bd7e8f56c
Parents: 2154c75
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 26 14:18:06 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 26 14:18:06 2014 +0100
----------------------------------------------------------------------
.../registry/retrieve/RegistryRetriever.java | 43 +++++++++++++-------
.../server/appmaster/SliderAppMaster.java | 3 +-
.../YarnRegistryViewForProviders.java | 17 +++++---
.../TestStandaloneYarnRegistryAM.groovy | 2 +-
.../providers/hbase/HBaseProviderService.java | 12 ++++--
5 files changed, 51 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index 1d11ef3..df1d675 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -25,8 +25,10 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.exceptions.RegistryIOException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.common.tools.SliderUtils;
@@ -68,25 +70,37 @@ public class RegistryRetriever {
this.internalConfigurationURL = internalConfigurationURL;
}
- public RegistryRetriever(ServiceRecord record) throws InvalidRecordException {
+ /**
+ * Retrieve from a service by locating the
+ * exported {@link CustomRegistryConstants#PUBLISHER_CONFIGURATIONS_API}
+ * and working off it.
+ * @param record service record
+ * @throws RegistryIOException the address type of the endpoint does
+ * not match that expected (i.e. not a list of URLs), missing endpoint...
+ */
+ public RegistryRetriever(ServiceRecord record) throws RegistryIOException {
Endpoint internal = record.getInternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
- List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
- internal);
- if (addresses != null && !addresses.isEmpty()) {
- internalConfigurationURL = addresses.get(0);
- } else {
- internalConfigurationURL = "";
+ String url = null;
+ if (internal != null) {
+ List<String> addresses = RegistryTypeUtils.retrieveAddressesUriType(
+ internal);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
}
+ internalConfigurationURL = url;
Endpoint external = record.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API);
-
- addresses = RegistryTypeUtils.retrieveAddressesUriType(external);
- if (addresses != null && !addresses.isEmpty()) {
- externalConfigurationURL = addresses.get(0);
- } else {
- externalConfigurationURL = "";
+ url = null;
+ if (external != null) {
+ List<String> addresses =
+ RegistryTypeUtils.retrieveAddressesUriType(external);
+ if (addresses != null && !addresses.isEmpty()) {
+ url = addresses.get(0);
+ }
}
+ externalConfigurationURL = url;
}
/**
@@ -166,7 +180,8 @@ public class RegistryRetriever {
@Override
public String toString() {
return super.toString()
- + " - external " + externalConfigurationURL;
+ + ": internal URL: \"" + internalConfigurationURL
+ + "\"; external \"" + externalConfigurationURL +"\"";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 2d3e5e4..2625d74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -901,7 +901,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
yarnRegistryOperations = new YarnRegistryViewForProviders(
registryOperations, service_user_name,
SliderKeys.APP_TYPE,
- instanceName);
+ instanceName,
+ appAttemptID);
providerService.bindToYarnRegistry(yarnRegistryOperations);
sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index d699279..e5b150a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -18,6 +18,7 @@
package org.apache.slider.server.services.yarnregistry;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
@@ -35,14 +36,23 @@ public class YarnRegistryViewForProviders {
private final String sliderServiceclass;
private final String instanceName;
+ private final ApplicationAttemptId applicationAttemptId;
private ServiceRecord selfRegistration;
public YarnRegistryViewForProviders(RegistryOperations registryOperations,
- String user, String sliderServiceclass, String instanceName) {
+ String user,
+ String sliderServiceclass,
+ String instanceName,
+ ApplicationAttemptId applicationAttemptId) {
this.registryOperations = registryOperations;
this.user = user;
this.sliderServiceclass = sliderServiceclass;
this.instanceName = instanceName;
+ this.applicationAttemptId = applicationAttemptId;
+ }
+
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId;
}
public String getUser() {
@@ -100,7 +110,6 @@ public class YarnRegistryViewForProviders {
registryOperations.create(path, record, CreateFlags.OVERWRITE);
}
-
/**
* Add a service under a path
* @param username user
@@ -117,10 +126,8 @@ public class YarnRegistryViewForProviders {
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
-
}
-
/**
* Add a service under a path for the current user
* @param serviceClass service class to use under ~user
@@ -136,10 +143,8 @@ public class YarnRegistryViewForProviders {
user, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
-
}
-
public void rmComponent(String componentName) throws IOException {
String path = RegistryOperationUtils.componentPath(
user, sliderServiceclass, instanceName,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 4cd662b..d081676 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -139,7 +139,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
RegistryOperationsClient registryOperationsClient =
registryService as RegistryOperationsClient
try {
- def yarnRegistryDump = registryOperationsClient.dumpPath()
+ def yarnRegistryDump = registryOperationsClient.dumpPath(false)
log.info("yarn service registry: \n${yarnRegistryDump}\n")
} catch (IOException ignored) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46fb5dbd/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 3c059d9..1760e72 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
@@ -254,16 +255,19 @@ public class HBaseProviderService extends AbstractProviderService
private void registerHBaseServiceEntry() throws IOException {
-
String name = amState.getApplicationName() ;
ServiceRecord serviceRecord = new ServiceRecord();
-
+ // bond lifespan to the application
+ serviceRecord.yarn_id = yarnRegistry.getApplicationAttemptId()
+ .getApplicationId().toString();
+ serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION;
try {
- URL configURL = new URL(amWebAPI, SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
+ URL configURL = new URL(amWebAPI,
+ SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.restEndpoint(
- CustomRegistryConstants.PUBLISHER_REST_API,
+ CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API,
configURL.toURI()));
} catch (URISyntaxException e) {
log.warn("failed to create config URL: {}", e, e);
[09/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f67b5507
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f67b5507
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f67b5507
Branch: refs/heads/develop
Commit: f67b5507b7230a025e7c54064e2614258f009116
Parents: 6e9380c a8dfc67
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 12 09:59:58 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 12 09:59:58 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/pom.xml | 52 +++--
.../command-logger/slider-pkg/appConfig.json | 2 +-
.../command-logger/slider-pkg/metainfo.xml | 2 +-
app-packages/hbase/metainfo.xml | 2 +-
slider-assembly/pom.xml | 20 ++
slider-assembly/src/assembly/slider-bin.xml | 8 +
slider-core/pom.xml | 28 +++
.../org/apache/slider/api/InternalKeys.java | 6 +-
.../org/apache/slider/client/SliderClient.java | 12 +-
.../org/apache/slider/common/SliderKeys.java | 1 +
.../apache/slider/common/tools/SliderUtils.java | 4 +
.../slider/core/build/InstanceBuilder.java | 28 +--
.../apache/slider/providers/ProviderUtils.java | 33 ++-
.../providers/agent/AgentClientProvider.java | 41 ++--
.../providers/agent/AgentProviderService.java | 19 +-
.../web/rest/agent/ExecutionCommand.java | 16 ++
.../appmaster/web/rest/agent/StatusCommand.java | 13 ++
.../test_command_log/appConfig.json | 2 +-
.../providers/agent/TestBuildBasicAgent.groovy | 19 --
.../agent/TestAgentClientProvider2.java | 88 +++++++
.../agent/TestAgentProviderService.java | 230 ++++++++++++++++++-
slider-funtest/pom.xml | 7 +-
.../org/apache/chaos/remote/BuildPidPath.groovy | 54 +++++
.../org/apache/chaos/remote/Clustat.groovy | 64 ++++++
.../apache/chaos/remote/JschToCommonsLog.groovy | 78 +++++++
.../chaos/remote/RemoteDaemonOperations.groovy | 60 +++++
.../org/apache/chaos/remote/RemoteServer.groovy | 188 +++++++++++++++
.../org/apache/chaos/remote/SshCommands.groovy | 190 +++++++++++++++
.../funtest/lifecycle/AMFailuresIT.groovy | 191 +++++++++++++++
29 files changed, 1353 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e10eb50,b2d10f6..baa413e
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -1186,10 -1157,12 +1181,12 @@@ public class SliderClient extends Abstr
clusterDirectory,
generatedConfDirPath,
clusterSecure
- );
+ );
- // now add the image if it was set
+ // TODO: consider supporting apps that don't have an image path
+ Path imagePath =
+ SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f67b5507/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
[11/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3ed6b2df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3ed6b2df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3ed6b2df
Branch: refs/heads/develop
Commit: 3ed6b2dfa6f7e0db8e09824cb87febd1ff1fb8f9
Parents: b0eac88 e7c008a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 16 14:32:22 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 16 14:32:22 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 5 ++-
.../common/params/AbstractActionArgs.java | 3 +-
.../common/params/ActionRegistryArgs.java | 33 ++++++++++-------
.../apache/slider/common/params/CommonArgs.java | 5 +--
.../slider/core/exceptions/UsageException.java | 34 ++++++++++++++++++
.../slider/core/launch/AbstractLauncher.java | 36 ++++++++++++++++---
.../server/appmaster/SliderAppMaster.java | 4 ++-
.../appmaster/monkey/ChaosKillContainer.java | 13 ++++++-
.../slider/server/appmaster/state/AppState.java | 5 +--
.../standalone/TestStandaloneAgentAM.groovy | 10 +++++-
.../slider/client/TestClientBadArgs.groovy | 14 ++++++++
.../slider/client/TestClientBasicArgs.groovy | 3 +-
.../registry/curator/TestLocalRegistry.groovy | 17 ++++-----
.../model/mock/BaseMockAppStateTest.groovy | 19 ++++++++++
.../model/mock/MockApplicationAttemptId.groovy | 8 +++++
.../model/mock/MockApplicationId.groovy | 9 ++++-
.../appmaster/model/mock/MockContainerId.groovy | 38 +++++++++++---------
.../appmaster/model/mock/MockFactory.groovy | 2 +-
.../appmaster/model/mock/MockYarnCluster.groovy | 7 +++-
.../model/monkey/TestMockMonkey.groovy | 14 ++++++++
.../web/view/TestContainerStatsBlock.groovy | 21 +++++------
.../appmaster/web/view/TestIndexBlock.groovy | 11 +++---
.../apache/slider/test/SliderTestUtils.groovy | 2 ++
23 files changed, 242 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3ed6b2df/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index baa413e,3df5b18..8c07c5d
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -2379,11 -2328,9 +2379,10 @@@ public class SliderClient extends Abstr
outputConfig(publishedConfiguration, registryArgs);
} else {
// it's an unknown command
- throw new BadCommandArgumentsException(
- "Bad command arguments for " + ACTION_REGISTRY + " " +
- registryArgs);
+ log.info(ActionRegistryArgs.USAGE);
+ return EXIT_USAGE;
}
+// JDK7
} catch (FileNotFoundException e) {
log.info("{}", e.toString());
log.debug("{}", e, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3ed6b2df/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3ed6b2df/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3ed6b2df/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
[37/50] [abbrv] git commit: SLIDER-149/YARN-913 persistence policy
set via set() method
Posted by st...@apache.org.
SLIDER-149/YARN-913 persistence policy set via set() method
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/330579e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/330579e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/330579e4
Branch: refs/heads/develop
Commit: 330579e47f8bc3cd520d23a0e3c4bcae10d2ee45
Parents: 27381a6
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 6 21:42:39 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 6 21:42:39 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 5 +++--
.../slider/server/appmaster/SliderAppMaster.java | 16 ++++++++++------
.../standalone/TestStandaloneYarnRegistryAM.groovy | 7 ++++---
.../providers/hbase/HBaseProviderService.java | 9 ++++++---
4 files changed, 23 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 602d6cf..b3365a7 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
@@ -2789,9 +2790,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.getYarn_id());
+ log.info("{}", instance.get(YarnRegistryAttributes.YARN_ID, ""));
} else {
- log.info("{}: ", instance.getYarn_id());
+ log.info("{}: ", instance.get(YarnRegistryAttributes.YARN_ID, ""));
logEndpoints(instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 6f7275c..5f7ebe6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
@@ -931,8 +932,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
- serviceRecord.putYarn_id(appid.toString());
- serviceRecord.putYarn_persistence(PersistencePolicies.APPLICATION);
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID, appid.toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -964,8 +966,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and a shorter lived binding to the app
String attempt = appAttemptID.toString();
ServiceRecord attemptRecord = new ServiceRecord(serviceRecord);
- attemptRecord.putYarn_id(attempt);
- attemptRecord.putYarn_persistence(PersistencePolicies.APPLICATION_ATTEMPT);
+ attemptRecord.set(YarnRegistryAttributes.YARN_ID, attempt);
+ attemptRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION_ATTEMPT);
yarnRegistryOperations.putComponent(
RegistryPathUtils.encodeYarnID(attempt),
serviceRecord);
@@ -1008,9 +1011,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info("Registering component {}", id);
String cid = RegistryPathUtils.encodeYarnID(id.toString());
ServiceRecord container = new ServiceRecord();
- container.putYarn_id(cid);
+ container.set(YarnRegistryAttributes.YARN_ID, cid);
container.description = description;
- container.putYarn_persistence(PersistencePolicies.CONTAINER);
+ container.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.CONTAINER);
try {
yarnRegistryOperations.putComponent(cid, container);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 7e74625..62b39f0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -329,9 +329,10 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def listedInstance = client.actionRegistryList(registryArgs)
def resolvedRecord = listedInstance[0]
- assert resolvedRecord[YarnRegistryAttributes.YARN_ID] == serviceRecord.getYarn_id()
- assert resolvedRecord[YarnRegistryAttributes.YARN_PERSISTENCE] == serviceRecord.getYarn_persistence()
-
+ assert resolvedRecord[YarnRegistryAttributes.YARN_ID] ==
+ serviceRecord[YarnRegistryAttributes.YARN_ID]
+ assert resolvedRecord[YarnRegistryAttributes.YARN_PERSISTENCE] ==
+ serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
// listconf
registryArgs.list = false;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/330579e4/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index ae9084b..6375d6f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
@@ -257,9 +258,11 @@ public class HBaseProviderService extends AbstractProviderService
String name = amState.getApplicationName() ;
ServiceRecord serviceRecord = new ServiceRecord();
// bond lifespan to the application
- serviceRecord.putYarn_id(yarnRegistry.getApplicationAttemptId()
- .getApplicationId().toString());
- serviceRecord.putYarn_persistence(PersistencePolicies.APPLICATION);
+ serviceRecord.set(YarnRegistryAttributes.YARN_ID,
+ yarnRegistry.getApplicationAttemptId()
+ .getApplicationId().toString());
+ serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.APPLICATION);
try {
URL configURL = new URL(amWebAPI,
SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
[34/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c51f2f0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c51f2f0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c51f2f0e
Branch: refs/heads/develop
Commit: c51f2f0eac6a7938e1f0acc27a4f5df90bb88339
Parents: 733745e 70b4b75
Author: Steve Loughran <st...@apache.org>
Authored: Sun Oct 5 19:56:47 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Oct 5 19:56:47 2014 -0700
----------------------------------------------------------------------
.../accumulo/configuration/accumulo-site.xml | 19 +-
app-packages/accumulo/configuration/global.xml | 7 +-
app-packages/accumulo/pom.xml | 2 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 4 +-
.../slider/funtest/accumulo/CertUtil.java | 50 +++-
.../app-pkg-template/appConfig-default.json | 2 +-
.../command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/hbase/pom.xml | 2 +-
app-packages/memcached/appConfig-default.json | 2 +-
app-packages/storm-win/appConfig-default.json | 2 -
app-packages/storm-win/metainfo.xml | 12 +-
.../storm-win/package/scripts/params.py | 2 -
.../storm-win/package/scripts/service.py | 1 +
.../storm-win/package/scripts/status_params.py | 2 -
.../storm-win/package/templates/config.yaml.j2 | 9 -
app-packages/storm/appConfig-default.json | 4 +-
app-packages/storm/metainfo.xml | 23 +-
app-packages/storm/package/scripts/params.py | 2 -
.../storm/package/templates/config.yaml.j2 | 9 -
app-packages/storm/resources-default.json | 4 -
pom.xml | 2 +
slider-assembly/src/main/scripts/slider | 2 +-
.../org/apache/slider/client/SliderClient.java | 77 +++--
.../apache/slider/common/SliderXmlConfKeys.java | 4 +-
.../apache/slider/core/conf/AggregateConf.java | 20 ++
.../apache/slider/core/conf/MapOperations.java | 16 +-
.../slider/core/exceptions/SliderException.java | 6 +-
.../TriggerClusterTeardownException.java | 16 +-
.../providers/AbstractClientProvider.java | 9 +-
.../providers/AbstractProviderService.java | 21 +-
.../apache/slider/providers/ProviderCore.java | 9 +-
.../apache/slider/providers/ProviderUtils.java | 6 +-
.../providers/agent/AgentProviderService.java | 28 +-
.../providers/agent/ComponentInstanceState.java | 2 +-
.../slideram/SliderAMClientProvider.java | 28 ++
.../slideram/SliderAMProviderService.java | 8 -
.../apache/slider/server/appmaster/AMUtils.java | 5 +-
.../server/appmaster/SliderAppMaster.java | 298 +++++++++++--------
.../appmaster/actions/ActionStopQueue.java | 8 +-
.../appmaster/actions/ActionStopSlider.java | 116 +++++++-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../server/appmaster/actions/QueueExecutor.java | 10 +
.../server/appmaster/actions/QueueService.java | 22 +-
.../slider/server/appmaster/state/AppState.java | 13 +-
.../server/appmaster/state/RoleStatus.java | 20 ++
.../workflow/WorkflowSequenceService.java | 8 +-
.../agent/actions/TestActionStatus.groovy | 16 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 99 ++++++
.../TestFreezeThawMasterlessAM.groovy | 92 ------
.../standalone/TestBuildStandaloneAM.groovy | 22 +-
.../standalone/TestStandaloneAMDestroy.groovy | 20 +-
.../TestStandaloneAMMonkeyRestart.groovy | 12 +-
.../standalone/TestStandaloneAMRestart.groovy | 77 ++++-
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
.../slider/providers/agent/TestAgentEcho.groovy | 18 +-
.../TestMockAppStateContainerFailure.groovy | 1 -
.../slider/test/YarnMiniClusterTestBase.groovy | 37 ++-
.../accumulo/live/TestAccFreezeThaw.groovy | 7 +-
.../providers/hbase/HBaseProviderService.java | 23 --
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
61 files changed, 857 insertions(+), 490 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --cc app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 7ddaf9f,5ebce0a..fd8af9a
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@@ -167,11 -167,13 +167,12 @@@ class AccumuloBasicIT extends AccumuloA
public static String getMonitorUrl(SliderClient sliderClient, String clusterName) {
int tries = 5
+ Exception caught;
while (true) {
try {
- CuratorServiceInstance<ServiceInstanceData> instance =
- sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE, clusterName)
- ServiceInstanceData serviceInstanceData = instance.payload
- RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
+ ServiceRecord instance =
+ sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName)
+ RegistryRetriever retriever = new RegistryRetriever(instance)
PublishedConfiguration configuration = retriever.retrieveConfiguration(
retriever.getConfigurations(true), "quicklinks", true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 2625d74,5676f3f..a259e42
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@@ -50,14 -51,9 +51,15 @@@ import org.apache.hadoop.yarn.client.ap
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+ import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@@ -357,8 -358,8 +362,9 @@@ public class SliderAppMaster extends Ab
private final QueueService actionQueues = new QueueService();
private String agentOpsUrl;
private String agentStatusUrl;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
private FsDelegationTokenManager fsDelegationTokenManager;
+ private RegisterApplicationMasterResponse amRegistrationData;
/**
* Service Constructor
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 13c876e,a14a14d..b6ac657
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@@ -30,6 -30,6 +30,7 @@@ import org.apache.slider.common.params.
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
++import org.apache.slider.core.registry.YarnAppListClient
import org.junit.Test
import static org.apache.slider.common.params.Arguments.ARG_COMP_OPT
@@@ -66,7 -66,9 +67,8 @@@ class TestBuildStandaloneAM extends Age
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- def serviceRegistryClient = sliderClient.yarnAppListClient
-
+ // verify the YARN registry doesn't know of it
- def serviceRegistryClient = sliderClient.YARNRegistryClient
++ YarnAppListClient serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
----------------------------------------------------------------------
[25/50] [abbrv] git commit: SLIDER-149 REST registry interface;
in sync with latest changes of the code
Posted by st...@apache.org.
SLIDER-149 REST registry interface; in sync with latest changes of the code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/da814b74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/da814b74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/da814b74
Branch: refs/heads/develop
Commit: da814b7440ea8aeb21b9811df53b93b0eddece33
Parents: 44e6165
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 25 20:43:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 25 20:43:50 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 11 +-
.../core/exceptions/ExceptionConverter.java | 16 +-
.../docstore/PublishedConfiguration.java | 4 +-
.../core/registry/info/RegisteredDocument.java | 33 ---
.../core/registry/info/RegisteredEndpoint.java | 1 +
.../core/registry/info/RegistryFields.java | 34 ---
.../core/registry/info/RegistryNaming.java | 51 -----
.../slider/core/registry/info/RegistryView.java | 1 +
.../server/appmaster/SliderAppMaster.java | 21 +-
.../web/rest/registry/PathEntryResource.java | 51 +++++
.../web/rest/registry/RegistryResource.java | 81 ++++---
.../server/services/curator/CuratorHelper.java | 1 +
.../TestStandaloneYarnRegistryAM.groovy | 24 ++-
.../registry/curator/TestLocalRegistry.groovy | 164 --------------
.../curator/TestRegistryRestResources.groovy | 195 -----------------
.../curator/TestServiceInstanceSerDeser.groovy | 122 -----------
.../rest/registry/PathEntryMarshalling.groovy | 28 +++
.../registry/TestRegistryRestMarshalling.groovy | 47 ++++
.../registry/TestRegistryRestResources.groovy | 213 +++++++++++++++++++
slider-core/src/test/resources/log4j.properties | 3 +-
.../appmaster/web/rest/registry/sample.json | 76 +++++++
.../funtest/FunctionalHBaseClusterIT.groovy | 7 +-
.../minicluster/live/TestHBaseMaster.groovy | 2 +-
23 files changed, 526 insertions(+), 660 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e68a7cb..de27dcd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*;
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
@@ -2453,9 +2454,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.id);
+ log.info("{}", instance.yarn_id);
} else {
- log.info("{}: ", instance.id);
+ log.info("{}: ", instance.yarn_id);
logEndpoints(instance);
}
}
@@ -2599,7 +2600,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param serviceType service type
* @param id instance ID
* @return instance data
- * @throws UnknownApplicationInstanceException no match
+ * @throws UnknownApplicationInstanceException no path or service record
+ * at the end of the path
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
@@ -2609,8 +2611,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return getRegistryOperations().resolve(
servicePath(currentUser(),
serviceType, id));
+ // TODO JDK7 SWITCH
} catch (PathNotFoundException e) {
throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
+ } catch (NoRecordException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
index 98dc028..ed5ceb2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ExceptionConverter.java
@@ -20,6 +20,7 @@ package org.apache.slider.core.exceptions;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
+import org.apache.hadoop.fs.PathAccessDeniedException;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -39,17 +40,22 @@ public class ExceptionConverter {
public static IOException convertJerseyException(String targetURL,
UniformInterfaceException exception) {
+ IOException ioe = null;
ClientResponse response = exception.getResponse();
if (response != null) {
int status = response.getStatus();
+ if (status == 401) {
+ ioe = new PathAccessDeniedException(targetURL);
+ }
if (status >= 400 && status < 500) {
- FileNotFoundException fnfe =
- new FileNotFoundException(targetURL);
- fnfe.initCause(exception);
- return fnfe;
+ ioe = new FileNotFoundException(targetURL);
}
}
- return new IOException("Failed to GET " + targetURL + ": " + exception, exception);
+ if (ioe == null) {
+ ioe = new IOException("Failed to GET " + targetURL + ": " + exception);
+ }
+ ioe.initCause(exception);
+ return ioe;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index f76b93b..cbc46f0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -150,8 +150,8 @@ public class PublishedConfiguration {
/**
* Return the values as json string
- * @return
- * @throws IOException
+ * @return the JSON representation
+ * @throws IOException marshalling failure
*/
public String asJson() throws IOException {
ObjectMapper mapper = new ObjectMapper();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
deleted file mode 100644
index cde282b..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredDocument.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- * A registry document
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegisteredDocument {
- public String contentType;
- public String url;
- public String description;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
index 8c065e2..2351e06 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
@@ -32,6 +32,7 @@ import java.net.URL;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Deprecated
public class RegisteredEndpoint {
// standard types
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
deleted file mode 100644
index 441a7de..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryFields.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-/**
- * Fieldnames in registry entries
- */
-public class RegistryFields {
-
- public static final String ENDPOINTS = "endpoints";
- public static final String INTERNAL_VIEW = "internalView";
- public static final String EXTERNAL_VIEW = "externalView";
-
- public static final String PROTOCOL = "protocol";
- public static final String VALUE = "value";
- public static final String DESCRIPTION = "description";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java
deleted file mode 100644
index bd5cf27..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryNaming.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import java.util.Locale;
-
-/**
- * Logic for greating names from registry entries; lets the policy be changed
- * later
- */
-public class RegistryNaming {
-
- public static String SLIDER_INSTANCE_NAME_FORMAT = "%2$s";
-
- public static String createRegistryServiceType(String instanceName,
- String userName,
- String serviceName) {
- return serviceName;
- }
-
- public static String createRegistryName(String instanceName,
- String userName,
- String serviceName,
- int appId) {
- return String.format(Locale.ENGLISH,
- SLIDER_INSTANCE_NAME_FORMAT,
- userName,
- instanceName,
- serviceName,
- appId);
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
index bdf70a2..89b9c6e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
@@ -26,6 +26,7 @@ import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Deprecated
public class RegistryView {
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index df54cfa..2d3e5e4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -95,7 +95,6 @@ import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegistryNaming;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
@@ -887,7 +886,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @throws Exception
*/
private void registerServiceInstance(String instanceName,
- ApplicationId appid) throws Exception {
+ ApplicationId appid) throws IOException {
// the registry is running, so register services
@@ -896,14 +895,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
URL agentStatusURI = new URL(agentStatusUrl);
String serviceName = SliderKeys.APP_TYPE;
int id = appid.getId();
- String serviceType = RegistryNaming.createRegistryServiceType(
- instanceName,
- service_user_name,
- serviceName);
- String registryId =
- RegistryNaming.createRegistryName(instanceName, service_user_name,
- serviceName, id);
-
//Give the provider restricted access to the state, registry
setupInitialRegistryPaths();
@@ -916,9 +907,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
- String serviceID = appid.toString();
- serviceRecord.id = serviceID;
- serviceRecord.persistence = PersistencePolicies.APPLICATION;
+ serviceRecord.yarn_id = appid.toString();
+ serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION;
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -939,7 +929,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
agentStatusURI,
serviceRecord);
-
// store for clients
log.info("Service Record \n{}", serviceRecord);
yarnRegistryOperations.putService(service_user_name,
@@ -951,8 +940,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and a shorter lived binding to the app
String attempt = appAttemptID.toString();
ServiceRecord attemptRecord = new ServiceRecord(serviceRecord);
- attemptRecord.id = attempt;
- attemptRecord.persistence = PersistencePolicies.APPLICATION_ATTEMPT;
+ attemptRecord.yarn_id = attempt;
+ attemptRecord.yarn_persistence = PersistencePolicies.APPLICATION_ATTEMPT;
yarnRegistryOperations.putComponent(
RegistryPathUtils.encodeYarnID(attempt),
serviceRecord);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
new file mode 100644
index 0000000..8790871
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/PathEntryResource.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Representation of a path entry
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class PathEntryResource {
+
+ /**
+ * Child nodes: as the full path to each element
+ */
+ public List<String> nodes;
+
+ /**
+ * Service record: if null —there is no resolvable service
+ * record at this node.
+ */
+ public ServiceRecord service;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
index 67efa0d..23f7261 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -23,9 +23,12 @@ import org.apache.hadoop.fs.PathAccessDeniedException;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException;
+import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,10 +39,12 @@ import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
+import java.io.IOException;
import java.util.List;
/**
@@ -53,7 +58,6 @@ import java.util.List;
*
*/
@Singleton
-//@Path(RestPaths.SLIDER_PATH_REGISTRY)
public class RegistryResource {
protected static final Logger log =
LoggerFactory.getLogger(RegistryResource.class);
@@ -80,51 +84,80 @@ public class RegistryResource {
}
@GET
- public Response getRoot(@Context HttpServletRequest request) {
- return Response.ok("registry root").build();
+ @Produces({MediaType.APPLICATION_JSON})
+ public PathEntryResource getRoot(@Context HttpServletRequest request,
+ @Context UriInfo uriInfo) {
+ return lookup("/", request, uriInfo);
}
-
// {path:.*}
@Path(SERVICE_PATH)
@GET
@Produces({MediaType.APPLICATION_JSON})
- public Response lookup(
+ public PathEntryResource lookup(
@PathParam("path") String path,
@Context HttpServletRequest request,
@Context UriInfo uriInfo) {
- try {
init(request, uriInfo);
- List<RegistryPathStatus> list = registry.listFull(path);
- return Response.ok("found").build();
+ return resolvePath(path);
+ }
+
+ /**
+ * Do the actual processing of requests to responses; can be directly
+ * invoked for testing.
+ * @param path path to query
+ * @return the entry
+ * @throws WebApplicationException on any failure.
+ */
+ public PathEntryResource resolvePath(String path) throws
+ WebApplicationException {
+ try {
+ PathEntryResource pathEntry =
+ fromRegistry(path);
+ if (log.isDebugEnabled()) {
+ log.debug("Resolved:\n{}", pathEntry);
+ }
+ return pathEntry;
+ } catch (WebApplicationException e) {
+ // rethrow direct
+ throw e;
} catch (PathNotFoundException e) {
- throw new NotFoundException(path);
+ throw new NotFoundException("Not found: " + path);
} catch (AuthenticationFailedException e) {
throw new ForbiddenException(path);
} catch (PathAccessDeniedException e) {
throw new ForbiddenException(path);
} catch (Exception e) {
- return fromException(e);
+ log.error("Error during generation of response: {}", e, e);
+ throw new WebApplicationException(e);
}
}
+
/**
- * Handle an exception
- * @param e exception
- * @return a response to return
+ * Build from the registry, filling up the children and service records.
+ * If there is no service record at the end of the path, that entry is
+ * null
+ * @param registry registry operations
+ * @param path path to query
+ * @return the built up record
+ * @throws IOException problems
+ *
*/
- Response fromException(Exception e) {
- log.error("Error during generation of response: {}", e, e);
- if (e instanceof PathNotFoundException) {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
- if (e instanceof AuthenticationFailedException
- || e instanceof PathAccessDeniedException) {
- return Response.status(Response.Status.FORBIDDEN).build();
+ private PathEntryResource fromRegistry(String path) throws IOException {
+ PathEntryResource entry = new PathEntryResource();
+ try {
+ entry.service = registry.resolve(path);
+ } catch (NoRecordException e) {
+ // ignoring
+ log.debug("No record at {}", path);
+ } catch (InvalidRecordException e) {
+ // swallowing this exception, the sign of "no entry present"
+ // "nothing parseable"
+ log.warn("Failed to resolve {}: {}", path, e, e);
}
- return Response.serverError().build();
+ entry.nodes = registry.list(path);
+ return entry;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
index 9e6f53a..96d296d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
@@ -37,6 +37,7 @@ import org.apache.slider.server.services.registry.SliderRegistryService;
* This class creates a curator -but does not start or close it. That
* is the responsbility of the owner
*/
+@Deprecated
public class CuratorHelper extends Configured {
private final CuratorFramework curator;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 65bd077..4cd662b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -49,6 +49,12 @@ import org.apache.slider.server.appmaster.PublishedArtifacts
import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.junit.Test
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_ONEWAY_REST_API
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_SECURE_REST_API
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.AM_IPC_PROTOCOL
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.MANAGEMENT_REST_API
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.PUBLISHER_REST_API
+
/**
* work with a YARN registry
*/
@@ -61,7 +67,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
@Test
- public void testYarnRegistryAM() throws Throwable {
+ public void testStandaloneYarnRegistryAM() throws Throwable {
describe "create a masterless AM then perform YARN registry operations on it"
@@ -156,7 +162,19 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def serviceRecord = serviceRecords[0]
log.info(serviceRecord.toString())
- assert 2 <= serviceRecord.external.size()
+ assert serviceRecord.yarn_id != null;
+ def externalEndpoints = serviceRecord.external;
+ assert externalEndpoints.size() > 0
+
+ def am_ipc_protocol = AM_IPC_PROTOCOL
+ serviceRecord.getExternalEndpoint(am_ipc_protocol)
+ assert null != am_ipc_protocol;
+
+ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
+ assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ // internals
+ assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
+ assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
// hit the registry web page
def registryEndpoint = serviceRecord.getExternalEndpoint(
@@ -311,7 +329,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
describe registryArgs.toString()
def listedInstance = client.actionRegistryListYarn(registryArgs)
- assert listedInstance[0].id == serviceRecord.id
+ assert listedInstance[0].yarn_id == serviceRecord.yarn_id
// listconf
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
deleted file mode 100644
index 71496d6..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestLocalRegistry.groovy
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.core.registry.retrieve.RegistryRetriever
-import org.apache.slider.server.services.curator.CuratorHelper
-import org.apache.slider.server.services.curator.RegistryBinderService
-import org.apache.slider.core.registry.info.RegistryNaming
-import org.apache.slider.server.services.registry.SliderRegistryService
-import org.apache.slider.test.MicroZKCluster
-import org.apache.slider.test.SliderTestUtils
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
-class TestLocalRegistry {
- MicroZKCluster miniZK
- SliderRegistryService registryBinder
-
- @Before
- void setup() {
- miniZK = new MicroZKCluster()
- miniZK.createCluster()
-
- registryBinder = createRegistry()
- }
-
- def SliderRegistryService createRegistry( ) {
- def conf = new YarnConfiguration()
- CuratorHelper curatorHelper =
- new CuratorHelper(conf, miniZK.zkBindingString);
-
- SliderRegistryService registry
- registry = curatorHelper.createRegistryBinderService("/services");
- registry.init(conf)
- registry.start()
- return registry
- }
-
- @After
- void teardown() {
- registryBinder?.stop()
- miniZK?.close()
- }
-
-
- @Test
- public void testRegisterAndQuery() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance != null
- }
-
- @Test
- public void testRegisterAndList() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- registryBinder.register(SliderKeys.APP_TYPE, "instance2",
- new URL("http", "localhost", 8090, "/"),
- null)
- def instances = registryBinder.instanceIDs(SliderKeys.APP_TYPE)
- assert instances.size() ==2
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance != null
- }
-
- @Test
- public void testMultipleRegistryBinders() throws Throwable {
- registryBinder.register(SliderKeys.APP_TYPE, "instance3",
- new URL("http", "localhost", 80, "/"),
- null)
- registryBinder.register(SliderKeys.APP_TYPE, "instance2",
- new URL("http", "localhost", 8090, "/"),
- null)
- SliderRegistryService registry2 = createRegistry()
- SliderRegistryService registry3 = createRegistry()
- try {
- def instances = registry3.instanceIDs(SliderKeys.APP_TYPE)
- assert instances.size() == 2
- def instance = registryBinder.queryForInstance(
- SliderKeys.APP_TYPE,
- "instance3")
- assert instance.id == "instance3"
- assert instance.name == SliderKeys.APP_TYPE
- } finally {
- registry3.stop()
- registry2.stop()
- }
- }
-
- @Test
- public void testNamingPolicy() throws Throwable {
-
- String hobbitName = RegistryNaming.createRegistryServiceType("hobbiton",
- "bilbo",
- SliderKeys.APP_TYPE);
- String hobbitId =
- RegistryNaming.createRegistryName(
- "hobbiton",
- "bilbo",
- SliderKeys.APP_TYPE,
- 1);
- String mordorName = RegistryNaming.createRegistryServiceType("mordor",
- "bilbo",
- SliderKeys.APP_TYPE);
- String mordorId =
- RegistryNaming.createRegistryName(
- "mordor",
- "bilbo",
- SliderKeys.APP_TYPE,
- 1);
-
- // service have same name
- assert hobbitName == mordorName;
- assert mordorId != hobbitId;
- registryBinder.register(mordorName, mordorId,
- new URL("http", "localhost", 8090, "/"),
- new ServiceInstanceData())
- registryBinder.register(hobbitName, hobbitId,
- new URL("http", "localhost", 80, "/mordor"),
- new ServiceInstanceData())
- def mordorInstance = registryBinder.queryForInstance(
- mordorName,
- mordorId)
- assert mordorInstance.port == 8090
-
- def instances = registryBinder.listInstances(SliderKeys.APP_TYPE);
- SliderTestUtils.dumpRegistryInstances(instances)
- assert instances.size() == 2
- def instance1 = instances[0]
- def payload1 = instance1.payload
-
- // now set up a registry retriever
- RegistryRetriever retriever = new RegistryRetriever(payload1)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
deleted file mode 100644
index 0b77adc..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import com.sun.jersey.api.client.Client
-import com.sun.jersey.api.client.ClientResponse
-import com.sun.jersey.api.client.UniformInterfaceException
-import com.sun.jersey.api.client.WebResource
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.curator.x.discovery.ServiceType
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.providers.agent.AgentTestBase
-import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.curator.CuratorServiceInstances
-import org.apache.slider.core.registry.info.RegistryNaming
-import org.junit.Test
-
-import javax.ws.rs.core.MediaType
-
-import static org.apache.slider.common.params.Arguments.ARG_OPTION
-import static org.apache.slider.providers.agent.AgentKeys.*
-import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
-
-@CompileStatic
-@Slf4j
-class TestRegistryRestResources extends AgentTestBase {
-
- public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
- public static final String WADL = "vnd.sun.wadl+xml"
- public static final String CLUSTERNAME = "testregistryws"
-
-
- private String id(String instanceName) {
-
- RegistryNaming.createRegistryName(
- instanceName,
- UserGroupInformation.getCurrentUser().getUserName(),
- SliderKeys.APP_TYPE,
- 1);
- }
-
-
- @Test
- public void testRestURIs() throws Throwable {
- def clustername = CLUSTERNAME
- createMiniCluster(
- clustername,
- configuration,
- 1,
- 1,
- 1,
- true,
- false)
- Map<String, Integer> roles = [:]
- File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- File app_def_path = new File(app_def_pkg_path)
- String agt_ver = "version"
- File agt_ver_path = new File(slider_core, agt_ver)
- String agt_conf = "agent.ini"
- File agt_conf_path = new File(slider_core, agt_conf)
- assert app_def_path.exists()
- assert agt_ver_path.exists()
- assert agt_conf_path.exists()
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
- roles,
- [
- ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, toURIArg(app_def_path),
- ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
- ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
- ],
- true, true,
- true)
- SliderClient sliderClient = launcher.service
- def report = waitForClusterLive(sliderClient)
- def trackingUrl = report.trackingUrl
- log.info("tracking URL is $trackingUrl")
- def registry_url = appendToURL(trackingUrl, REGISTRY_URI)
-
-
- def status = dumpClusterStatus(sliderClient, "agent AM")
- def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
- if (liveURL) {
- registry_url = appendToURL(liveURL, REGISTRY_URI)
-
- }
-
- log.info("Registry is $registry_url")
- log.info("stacks is ${liveURL}stacks")
- log.info("conf is ${liveURL}conf")
-
-
- //WS get
- Client client = createTestClient();
-
- WebResource webResource = client.resource(registry_url);
- List<String> serviceList = webResource.type(MediaType.APPLICATION_JSON)
- .get(List.class);
- log.info("service list: {}", serviceList)
- assert serviceList.contains(SliderKeys.APP_TYPE)
-
- // test the available GET URIs
- webResource = client.resource(
- appendToURL(registry_url, RestPaths.REGISTRY_SERVICE));
-
- ClientResponse response = webResource.type(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- def responseStr = response.getEntity(String.class)
- log.info("response is " + responseStr)
-
- "{\"names\":[\"${SliderKeys.APP_TYPE}\"]}".equals(responseStr)
-
- webResource = client.resource(
- appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}"));
- CuratorServiceInstances<ServiceInstanceData> services = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstances.class);
- assert services.services.size() == 1
- CuratorServiceInstance<ServiceInstanceData> service = services.services.get(0)
- validateService(service)
-
- webResource = client.resource(
- appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/"+id(
- clustername)));
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- validateService(service)
-
- webResource = client.resource(
- appendToURL(
- registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/${SliderKeys.APP_TYPE}"));
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- validateService(service)
-
- // some negative tests...
- webResource = client.resource(
- appendToURL(registry_url, "${RestPaths.REGISTRY_SERVICE}/dummy"))
- services = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstances.class);
- assert services.services.size() == 0
-
- try {
- webResource = client.resource(appendToURL(registry_url,
- "${RestPaths.REGISTRY_SERVICE}/${SliderKeys.APP_TYPE}/testregistryws99"));
-
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- fail("should throw an exception for a 404 response....")
- } catch (UniformInterfaceException e) {
- assert e.response.getStatus() == 404
- }
-
- try {
- webResource = client.resource(
- appendToURL(registry_url, "${RestPaths.REGISTRY_ANYSERVICE}/dummy"));
-
- service = webResource.type(MediaType.APPLICATION_JSON)
- .get(CuratorServiceInstance.class);
- fail("should throw an exception for a 404 response....")
- } catch (UniformInterfaceException e) {
- assert e.response.getStatus() == 404
- }
- }
-
- private void validateService(CuratorServiceInstance service) {
- assert service.name.equals(SliderKeys.APP_TYPE)
- assert service.serviceType == ServiceType.DYNAMIC
- assert service.id.contains(CLUSTERNAME)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
deleted file mode 100644
index 6a57cd9..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestServiceInstanceSerDeser.groovy
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.registry.curator
-
-import groovy.util.logging.Slf4j
-import org.apache.curator.x.discovery.ServiceInstance
-import org.apache.curator.x.discovery.ServiceInstanceBuilder
-import org.apache.curator.x.discovery.UriSpec
-import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.junit.Test
-
-/**
- * Here to identify why curator was complaining about deserializing
- * its own published artifacts.
- *
- * ... its a Jackson version thing
- */
-@Slf4j
-class TestServiceInstanceSerDeser {
-
- JsonSerDeser<ServiceInstance> serializer = new JsonSerDeser<>(
- ServiceInstance.class)
- JsonSerDeser<CuratorServiceInstance> deser = new JsonSerDeser<>(
- CuratorServiceInstance.class)
-
- @Test
- public void testDefault() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.name("defined")
- buildAndRoundTrip("testDefault", builder)
- }
-
- @Test
- public void testEmpty() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address(null).id("").name("").port(0).uriSpec(null)
-
- buildAndRoundTrip("testEmpty", builder)
- }
-
- @Test
- public void testFilled() throws Throwable {
- def builder = ServiceInstance.builder()
-
- builder.id("service").name("name").port(0)
- .uriSpec(new UriSpec("http:{}:{}"))
-
-
- buildAndRoundTrip("testFilled", builder)
- }
-
- @Test
- public void testPayload() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address(null).id("testPayload").name("").port(0).uriSpec(null)
-
- ServiceInstanceData data = new ServiceInstanceData()
- data.serviceType = "testPayload"
- data.externalView.documentsURL = "http://documents"
- builder.payload(data)
-
- def instance = buildAndRoundTrip("", builder)
-
- def payload = instance.payload as ServiceInstanceData
- log.info("payload = $payload")
- assert payload.externalView.documentsURL == "http://documents"
- }
-
- @Test
- public void testHackedDeser() throws Throwable {
- def builder = ServiceInstance.builder()
- builder.address("localhost")
- builder.id("service").name("name").port(8080)
- .uriSpec(new UriSpec("http:{}:{}"))
- .sslPort(443)
- ServiceInstanceData data = new ServiceInstanceData()
- data.externalView.documentsURL = "http://documents"
-
- builder.payload(data)
- log.info("Test: testHackedDeser")
- String json = serialize(builder)
- CuratorServiceInstance<ServiceInstanceData> curatorInstance = deser.fromJson(json)
- log.info("resolved =$curatorInstance")
- def payload = curatorInstance.payload
- log.info("payload = $payload")
- assert payload.externalView.documentsURL == "http://documents"
-
- }
-
- def buildAndRoundTrip(String testname, ServiceInstanceBuilder builder) {
- log.info("Test: $testname")
- String json = serialize(builder)
-
- return serializer.fromJson(json)
- }
-
- def String serialize(ServiceInstanceBuilder builder) {
- ServiceInstance<ServiceInstanceData> instance = builder.build()
- def json = serializer.toJson(instance)
- log.info(json)
- return json
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
new file mode 100644
index 0000000..3883b25
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry
+
+import org.apache.hadoop.yarn.registry.client.binding.JsonSerDeser
+
+class PathEntryMarshalling
+ extends JsonSerDeser<PathEntryResource> {
+ public PathEntryMarshalling() {
+ super(PathEntryResource.class, new byte[0]);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
new file mode 100644
index 0000000..44c8f05
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry
+
+import groovy.transform.CompileStatic
+import org.junit.Test
+
+/**
+ * This test exists because Jersey appears to behave "oddly"
+ * when it comes to marshalling JSON, and some of the REST resources
+ * appear to have trouble.
+ *
+ * This test tries to isolate it
+ */
+@CompileStatic
+class TestRegistryRestMarshalling {
+
+ @Test
+ public void testDeser() throws Throwable {
+ PathEntryMarshalling pem = new PathEntryMarshalling();
+ def unmarshalled = pem.fromResource(
+ "/org/apache/slider/server/appmaster/web/rest/registry/sample.json")
+
+ def serviceRecord = unmarshalled.service
+ assert serviceRecord
+ assert serviceRecord.yarn_id !=null
+ assert serviceRecord.yarn_persistence != 0
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
new file mode 100644
index 0000000..877ba56
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry
+
+import com.sun.jersey.api.client.Client
+import com.sun.jersey.api.client.ClientResponse
+import com.sun.jersey.api.client.UniformInterfaceException
+import com.sun.jersey.api.client.WebResource
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.agent.AgentTestBase
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Test
+
+import javax.ws.rs.core.MediaType
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
+
+@CompileStatic
+@Slf4j
+class TestRegistryRestResources extends AgentTestBase {
+
+ public static final String REGISTRY_URI = RestPaths.SLIDER_PATH_REGISTRY;
+ public static final String WADL = "vnd.sun.wadl+xml"
+ public static final String CLUSTERNAME = "test-registry-rest-resources"
+
+
+ @Test
+ public void testRestURIs() throws Throwable {
+
+ def clustername = CLUSTERNAME
+ createMiniCluster(
+ clustername,
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+ Map<String, Integer> roles = [:]
+ File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ File app_def_path = new File(app_def_pkg_path)
+ String agt_ver = "version"
+ File agt_ver_path = new File(slider_core, agt_ver)
+ String agt_conf = "agent.ini"
+ File agt_conf_path = new File(slider_core, agt_conf)
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def registry_url = appendToURL(trackingUrl, REGISTRY_URI)
+
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_WEB_URL)
+ if (liveURL) {
+ registry_url = appendToURL(liveURL, REGISTRY_URI)
+
+ }
+
+ log.info("Registry is $registry_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+ //WS get
+ Client client = createTestClient();
+
+ WebResource webResource = client.resource(registry_url);
+
+ def jsonType = webResource.type(MediaType.APPLICATION_JSON)
+ PathEntryResource entryResource = jsonType
+ .get(PathEntryResource.class);
+ processResponse("/", jsonType.get(ClientResponse.class))
+ assert entryResource.nodes != null;
+ assert entryResource.service == null;
+
+ // test the available GET URIs
+ def userhome = RegistryOperationUtils.homePathForCurrentUser()
+
+ def userServicesURL = appendToURL(registry_url,
+ userhome + RegistryConstants.PATH_USER_SERVICES)
+ webResource = client.resource(userServicesURL);
+
+
+ def jsonBuilder = webResource.type(MediaType.APPLICATION_JSON)
+ ClientResponse response = jsonBuilder.get(ClientResponse.class);
+ def responseStr = processResponse(userServicesURL, response)
+
+ assert responseStr.contains("\"nodes\"")
+ assert responseStr.contains("/${SliderKeys.APP_TYPE}")
+
+ entryResource = jsonBuilder.get(PathEntryResource.class)
+ assert entryResource.nodes.size() == 1;
+ assert entryResource.service == null;
+
+
+ def userServicesSlider = appendToURL(userServicesURL, SliderKeys.APP_TYPE)
+ webResource = client.resource(
+ userServicesSlider);
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ response = jsonBuilder.get(ClientResponse.class);
+ processResponse(userServicesURL, response)
+ entryResource = jsonBuilder.get(PathEntryResource.class)
+ assert entryResource.nodes.size() == 1;
+ assert entryResource.service == null;
+
+ def servicePath = entryResource.nodes[0]
+
+ // now work with a real instances
+
+ def instanceURL = appendToURL(userServicesSlider, clustername)
+ assert instanceURL.endsWith(servicePath)
+
+ webResource = client.resource(instanceURL);
+
+ // work with it via direct Jackson unmarshalling
+ responseStr = processResponse(instanceURL, webResource)
+ PathEntryMarshalling pem = new PathEntryMarshalling();
+ def unmarshalled = pem.fromJson(responseStr)
+ def r1 = unmarshalled.service
+ assert r1
+ assert r1.yarn_id != null
+ assert r1.yarn_persistence != 0
+
+ // and via the web resource AP
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ entryResource = jsonBuilder.get(PathEntryResource.class);
+
+ def serviceRecord = entryResource.service
+ assert serviceRecord != null;
+ assert serviceRecord.yarn_id != null;
+ def externalEndpoints = serviceRecord.external;
+ assert externalEndpoints.size() > 0
+
+ def am_ipc_protocol = AM_IPC_PROTOCOL
+ def epr = serviceRecord.getExternalEndpoint(am_ipc_protocol)
+ assert null != epr;
+
+ assert null != serviceRecord.getExternalEndpoint(MANAGEMENT_REST_API)
+ assert null != serviceRecord.getExternalEndpoint(PUBLISHER_REST_API)
+ // internals
+ assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
+ assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
+
+ // negative tests...
+ try {
+ webResource = client.resource(
+ appendToURL(registry_url, "/users/no-such-user"));
+ def clientResponse = webResource.get(ClientResponse.class)
+ assert 404 == clientResponse.status
+
+ def body = processResponse(userServicesURL, webResource)
+ jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
+ entryResource = jsonBuilder.get(PathEntryResource.class);
+
+
+ fail("should throw an exception for a 404 response, got " + body)
+ } catch (UniformInterfaceException e) {
+ assert e.response.status == 404
+ }
+ }
+
+ public String processResponse(String asURL, WebResource response) {
+ def jsonBuilder = response.type(MediaType.APPLICATION_JSON)
+ def clientResponse = jsonBuilder.get(ClientResponse.class);
+ return processResponse(asURL, clientResponse)
+ }
+
+ public String processResponse(String asURL, ClientResponse response) {
+ def responseStr = response.getEntity(String.class)
+ log.info(asURL + " ==>\n " + responseStr)
+ responseStr
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties
index c1a524d..2fe1e49 100644
--- a/slider-core/src/test/resources/log4j.properties
+++ b/slider-core/src/test/resources/log4j.properties
@@ -29,8 +29,7 @@ log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
log4j.logger.org.apache.slider=DEBUG
log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-
-
+log4j.logger.org.apache.hadoop.yarn.registry=DEBUG
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
new file mode 100644
index 0000000..b2d59fd
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
@@ -0,0 +1,76 @@
+{
+ "nodes": ["/users/example/services/org-apache-slider/test-registry-rest-resources/components"], "service": {
+ "registrationTime": 0,
+ "description": "Slider Application Master",
+ "yarn_id": "application_1411664296263_0001",
+ "yarn_persistence": 1,
+ "external": [
+ {
+ "api": "org.apache.slider.appmaster",
+ "addressType": "host/port",
+ "protocolType": "hadoop/protobuf",
+ "addresses": [
+ ["server", "63385"]
+ ]
+ },
+ {
+ "api": "org.apache.http.UI",
+ "addressType": "uri",
+ "protocolType": "webui",
+ "addresses": [
+ ["http://server:63391"]
+ ]
+ },
+ {
+ "api": "org.apache.slider.management",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["http://server:63391/ws/v1/slider/mgmt"]
+ ]
+ },
+ {
+ "api": "org.apache.slider.publisher",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["http://server:63391/ws/v1/slider/publisher"]
+ ]
+ },
+ {
+ "api": "org.apache.slider.registry",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["http://server:63391/ws/v1/slider/registry"]
+ ]
+ },
+ {
+ "api": "org.apache.slider.publisher.configurations",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["http://server:63391/ws/v1/slider/publisher/slider"]
+ ]
+ }
+ ],
+ "internal": [
+ {
+ "api": "org.apache.slider.agents.secure",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["https://server:63390/ws/v1/slider/agents"]
+ ]
+ },
+ {
+ "api": "org.apache.slider.agents.oneway",
+ "addressType": "uri",
+ "protocolType": "REST",
+ "addresses": [
+ ["https://server:63389/ws/v1/slider/agents"]
+ ]
+ }
+ ]
+}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
index 84e55f7..5137101 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
@@ -30,7 +30,7 @@ import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.registry.info.RegistryNaming
+
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.providers.hbase.HBaseConfigFileOptions
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -184,10 +184,7 @@ public class FunctionalHBaseClusterIT extends HBaseCommandTestBase
[ARG_LIST, ARG_NAME, "cluster-with-no-name"])
// how to work out the current service name?
- def name = RegistryNaming.createRegistryName(clustername,
- System.getProperty("user.name"),
- SliderKeys.APP_TYPE,
- 1)
+ def name = clustername
registry([ARG_LIST, ARG_VERBOSE, ARG_NAME, name])
registry([ARG_LISTCONF, ARG_NAME, name])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da814b74/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 9c53d2c..422ee8d 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -93,7 +93,7 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
assert hbaseInstances.size() == 1
ServiceRecord hbaseServiceData = hbaseInstances[0]
log.info "HBase service 0 == $hbaseServiceData"
- assert hbaseServiceData.id
+ assert hbaseServiceData.yarn_id
RegistryRetriever retriever = new RegistryRetriever(hbaseServiceData)
log.info retriever.toString()
[32/50] [abbrv] git commit: SLIDER149 in sync with YARN-913 changes
Posted by st...@apache.org.
SLIDER149 in sync with YARN-913 changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f058495a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f058495a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f058495a
Branch: refs/heads/develop
Commit: f058495a6c05a385518333d135b84d357780c7f6
Parents: 62335bf
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 30 17:42:22 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 30 17:42:22 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 +++---
.../YarnRegistryViewForProviders.java | 10 ++++-----
.../TestStandaloneYarnRegistryAM.groovy | 17 ++++++---------
.../registry/TestRegistryRestResources.groovy | 4 ++--
.../minicluster/live/TestHBaseMaster.groovy | 8 +++----
.../minicluster/live/TestTwoLiveClusters.groovy | 22 +++++++++-----------
6 files changed, 30 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 88d07b5..99896c8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*;
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
@@ -1455,7 +1455,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
public String getUsername() throws IOException {
- return RegistryOperationUtils.currentUser();
+ return RegistryUtils.currentUser();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index e5b150a..d0c891d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -20,7 +20,7 @@ package org.apache.slider.server.services.yarnregistry;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.api.CreateFlags;
@@ -104,7 +104,7 @@ public class YarnRegistryViewForProviders {
String serviceName,
String componentName,
ServiceRecord record) throws IOException {
- String path = RegistryOperationUtils.componentPath(
+ String path = RegistryUtils.componentPath(
user, serviceClass, serviceName, componentName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
@@ -122,7 +122,7 @@ public class YarnRegistryViewForProviders {
String serviceClass,
String serviceName,
ServiceRecord record) throws IOException {
- String path = RegistryOperationUtils.servicePath(
+ String path = RegistryUtils.servicePath(
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
@@ -139,14 +139,14 @@ public class YarnRegistryViewForProviders {
String serviceClass,
String serviceName,
ServiceRecord record) throws IOException {
- String path = RegistryOperationUtils.servicePath(
+ String path = RegistryUtils.servicePath(
user, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
}
public void rmComponent(String componentName) throws IOException {
- String path = RegistryOperationUtils.componentPath(
+ String path = RegistryUtils.componentPath(
user, sliderServiceclass, instanceName,
componentName);
registryOperations.delete(path, false);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 0155232..b9238f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -24,14 +24,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsClient
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
@@ -49,11 +48,7 @@ import org.apache.slider.server.appmaster.PublishedArtifacts
import org.apache.slider.server.appmaster.web.rest.RestPaths
import org.junit.Test
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_ONEWAY_REST_API
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.AGENT_SECURE_REST_API
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.AM_IPC_PROTOCOL
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.MANAGEMENT_REST_API
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.PUBLISHER_REST_API
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
/**
* work with a YARN registry
@@ -147,15 +142,15 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def self = currentUser()
- List<RegistryPathStatus> serviceTypes = registryService.listFull(homePathForUser(self))
+ def children = statChildren(registryService, homePathForUser(self));
+ Collection<RegistryPathStatus> serviceTypes = children.values()
dumpCollection(serviceTypes)
def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
- Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
+ Map<String, ServiceRecord> recordMap = extractServiceRecords(
registryService,
- recordsPath,
- registryService.listFull(recordsPath))
+ recordsPath);
def serviceRecords = recordMap.values();
dumpCollection(serviceRecords)
assert serviceRecords.size() == 1
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
index 9dd3092..e6d31fd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -25,7 +25,7 @@ import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
@@ -113,7 +113,7 @@ class TestRegistryRestResources extends AgentTestBase {
assert entryResource.service == null;
// test the available GET URIs
- def userhome = RegistryOperationUtils.homePathForCurrentUser()
+ def userhome = RegistryUtils.homePathForCurrentUser()
def userServicesURL = appendToURL(registry_url,
userhome + RegistryConstants.PATH_USER_SERVICES)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 422ee8d..84db30f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -20,7 +20,7 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
@@ -76,10 +76,10 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
// look up the registry entries for HBase
describe "service registry names"
- Map<String, ServiceRecord> records = RegistryOperationUtils.listServiceRecords(
+ Map<String, ServiceRecord> records = RegistryUtils.listServiceRecords(
client.registryOperations,
- RegistryOperationUtils.serviceclassPath(
- RegistryOperationUtils.homePathForCurrentUser(),
+ RegistryUtils.serviceclassPath(
+ RegistryUtils.homePathForCurrentUser(),
HBaseKeys.HBASE_SERVICE_TYPE
)
)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f058495a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index 01a67c7..bb652b3 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -21,7 +21,7 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
-import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
+import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.common.SliderKeys
@@ -86,19 +86,17 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
// registry instances def names = client.listRegistryNames(clustername)
describe "service registry names"
RegistryOperations registry = cluster2Client.registryOperations
- def home = RegistryOperationUtils.homePathForCurrentUser()
+ def home = RegistryUtils.homePathForCurrentUser()
- def userSliderInstancesPath = RegistryOperationUtils.serviceclassPath(
- RegistryOperationUtils.currentUser(), SliderKeys.APP_TYPE)
+ def userSliderInstancesPath = RegistryUtils.serviceclassPath(
+ RegistryUtils.currentUser(), SliderKeys.APP_TYPE)
- def names = RegistryOperationUtils.listServiceRecords(registry,
+ def names = RegistryUtils.listServiceRecords(registry,
userSliderInstancesPath)
dumpMap(names)
-
- def stats = registry.listFull(userSliderInstancesPath)
-
- dumpCollection(stats)
+ def statMap = RegistryUtils.statChildren(registry, userSliderInstancesPath)
+ assert statMap.size() == 2
List<String> instanceIds = sliderClient.listRegisteredSliderInstances()
dumpRegistryInstanceIDs(instanceIds)
@@ -112,11 +110,11 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
assert instances.size() == 2
- def hbaseServicePath = RegistryOperationUtils.serviceclassPath(
- RegistryOperationUtils.currentUser(),
+ def hbaseServicePath = RegistryUtils.serviceclassPath(
+ RegistryUtils.currentUser(),
HBaseKeys.HBASE_SERVICE_TYPE)
Map<String, ServiceRecord> hbaseInstances =
- RegistryOperationUtils.listServiceRecords(registry,
+ RegistryUtils.listServiceRecords(registry,
hbaseServicePath);
assert hbaseInstances.size() == 2
[07/50] [abbrv] git commit: YARN-913 ephemeral service records no
longer supported
Posted by st...@apache.org.
YARN-913 ephemeral service records no longer supported
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/edf72865
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/edf72865
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/edf72865
Branch: refs/heads/develop
Commit: edf728654754e47e360ae3b56b1bf8df8d24aa4a
Parents: 4eeed33
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 11 14:50:21 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 11 14:50:21 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 10 ++++----
.../YarnRegistryViewForProviders.java | 24 ++++++++------------
2 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/edf72865/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4b5e0b3..f924ef5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -970,8 +970,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and an ephemeral binding to the app
yarnRegistryOperations.putComponent(
RegistryPathUtils.encodeYarnID(appAttemptID.toString()),
- serviceRecord,
- false);
+ serviceRecord
+ );
}
@@ -987,7 +987,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if (registryOperations instanceof RMRegistryOperationsService) {
RMRegistryOperationsService rmRegOperations =
(RMRegistryOperationsService) registryOperations;
- rmRegOperations.createHomeDirectory(service_user_name);
+ rmRegOperations.initUserRegistryAsync(service_user_name);
}
}
@@ -1009,9 +1009,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceRecord container = new ServiceRecord(
cid,
description,
- PersistencePolicies.EPHEMERAL);
+ PersistencePolicies.CONTAINER);
try {
- yarnRegistryOperations.putComponent(cid, container, true);
+ yarnRegistryOperations.putComponent(cid, container);
} catch (IOException e) {
log.warn("Failed to register container {}/{}: {}",
id, description, e, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/edf72865/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index ef9c3a4..bea7f78 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -61,39 +61,35 @@ public class YarnRegistryViewForProviders {
/**
* Add a component under the slider name/entry
- * @param componentName
- * @param entry
- * @param ephemeral
+ * @param componentName component name
+ * @param record record to put
* @throws IOException
*/
public void putComponent(String componentName,
- ServiceRecord entry,
- boolean ephemeral) throws
+ ServiceRecord record) throws
IOException {
putComponent(sliderServiceclass, instanceName,
componentName,
- entry,
- ephemeral);
+ record
+ );
}
/**
* Add a component
- * @param componentName
- * @param record
- * @param ephemeral
+ * @param serviceClass service class to use under ~user
+ * @param componentName component name
+ * @param record record to put
* @throws IOException
*/
public void putComponent(String serviceClass,
String serviceName,
String componentName,
- ServiceRecord record,
- boolean ephemeral) throws IOException {
+ ServiceRecord record) throws IOException {
String path = BindingUtils.componentPath(
user, serviceClass, serviceName, componentName);
registryOperations.mkdir(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record,
- CreateFlags.OVERWRITE
- + (ephemeral ? CreateFlags.EPHEMERAL : 0));
+ CreateFlags.OVERWRITE);
}
[21/50] [abbrv] git commit: SLIDER-453 AM queue processing goes live
before init complete - SLIDER-149 extension
Posted by st...@apache.org.
SLIDER-453 AM queue processing goes live before init complete - SLIDER-149 extension
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8b2d4d5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8b2d4d5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8b2d4d5a
Branch: refs/heads/develop
Commit: 8b2d4d5a17e0f8f49a1eb04c4a5fe1e653f532aa
Parents: 77ca3c2
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:59:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:59:30 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8b2d4d5a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 2fa8342..754013a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1047,12 +1047,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Handler for {@link UnregisterComponentInstance}
*
* unregister a component. At the time this message is received,
- * the component may already been deleted from/never added to
- * the app state
+ * the component may not have been registered
* @param id the component
*/
public void unregisterComponent(ContainerId id) {
log.info("Unregistering component {}", id);
+ if (yarnRegistryOperations== null) {
+ log.warn("Processing unregister component event before initialization " +
+ "completed; init flag =" + initCompleted);
+ }
String cid = RegistryPathUtils.encodeYarnID(id.toString());
try {
yarnRegistryOperations.rmComponent(cid);
[15/50] [abbrv] git commit: SLIDER-149 in sync w/ YARN-913
Posted by st...@apache.org.
SLIDER-149 in sync w/ YARN-913
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2a288228
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2a288228
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2a288228
Branch: refs/heads/develop
Commit: 2a28822839d339780a3bebdf33ef136a3cd13adf
Parents: dbd1d3d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 14:54:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 14:54:01 2014 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 2 +-
.../java/org/apache/slider/core/persist/JsonSerDeser.java | 8 ++++----
.../services/yarnregistry/YarnRegistryViewForProviders.java | 4 ++--
.../agent/standalone/TestStandaloneYarnRegistryAM.groovy | 4 ++--
4 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a288228/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 93050a5..3e0b111 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2447,7 +2447,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
BindingUtils.serviceclassPath(BindingUtils.currentUser(),
serviceType);
RegistryPathStatus[] listDir;
- listDir = operations.listDir(serviceclassPath);
+ listDir = operations.list(serviceclassPath);
if (listDir.length == 0) {
throw new PathNotFoundException("records under "
+ serviceclassPath);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a288228/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index ab165b1..37526be 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -51,14 +51,14 @@ public class JsonSerDeser<T> {
private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class);
private static final String UTF_8 = "UTF-8";
- private final Class classType;
+ private final Class<T> classType;
private final ObjectMapper mapper;
/**
* Create an instance bound to a specific type
* @param classType
*/
- public JsonSerDeser(Class classType) {
+ public JsonSerDeser(Class<T> classType) {
this.classType = classType;
this.mapper = new ObjectMapper();
mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -74,7 +74,7 @@ public class JsonSerDeser<T> {
public T fromJson(String json)
throws IOException, JsonParseException, JsonMappingException {
try {
- return (T) (mapper.readValue(json, classType));
+ return mapper.readValue(json, classType);
} catch (IOException e) {
log.error("Exception while parsing json : " + e + "\n" + json, e);
throw e;
@@ -91,7 +91,7 @@ public class JsonSerDeser<T> {
public T fromFile(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
try {
- return (T) (mapper.readValue(jsonFile, classType));
+ return mapper.readValue(jsonFile, classType);
} catch (IOException e) {
log.error("Exception while parsing json file {}: {}", jsonFile, e);
throw e;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a288228/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index fc7ec2a..65ae420 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -87,7 +87,7 @@ public class YarnRegistryViewForProviders {
ServiceRecord record) throws IOException {
String path = BindingUtils.componentPath(
user, serviceClass, serviceName, componentName);
- registryOperations.mkdir(RegistryPathUtils.parentOf(path), true);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record,
CreateFlags.OVERWRITE);
}
@@ -108,7 +108,7 @@ public class YarnRegistryViewForProviders {
String path = BindingUtils.servicePath(
username, serviceClass, serviceName);
- registryOperations.mkdir(RegistryPathUtils.parentOf(path), true);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a288228/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index c7291a8..e88fbe8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -140,14 +140,14 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def registryService = client.registryOperations
def self = currentUser()
- RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
+ RegistryPathStatus[] serviceTypes = registryService.list(userPath(self))
dumpArray(serviceTypes)
def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
registryService,
- registryService.listDir(recordsPath))
+ registryService.list(recordsPath))
def serviceRecords = recordMap.values();
dumpCollection(serviceRecords)
assert serviceRecords.size() == 1
[19/50] [abbrv] git commit: SLIDER-149 java 5 template pain
Posted by st...@apache.org.
SLIDER-149 java 5 template pain
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b26863ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b26863ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b26863ab
Branch: refs/heads/develop
Commit: b26863ab63b20c6f008728dc0c5827a05b37c5cb
Parents: a06d5ed
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:53:57 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:53:57 2014 +0100
----------------------------------------------------------------------
.../main/java/org/apache/slider/core/persist/JsonSerDeser.java | 2 +-
.../slider/server/services/curator/RegistryBinderService.java | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b26863ab/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 97be41d..f77d6df 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -56,7 +56,7 @@ public class JsonSerDeser<T> {
/**
* Create an instance bound to a specific type
- * @param classType
+ * @param classType class type
*/
public JsonSerDeser(Class<T> classType) {
this.classType = classType;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b26863ab/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
index b3e2ff2..dbdcee4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -27,7 +27,6 @@ import org.apache.curator.x.discovery.ServiceInstance;
import org.apache.curator.x.discovery.ServiceInstanceBuilder;
import org.apache.curator.x.discovery.ServiceType;
import org.apache.curator.x.discovery.UriSpec;
-import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -56,8 +55,8 @@ public class RegistryBinderService<Payload> extends CuratorService {
private final Map<String, ServiceInstance<Payload>> entries =
new HashMap<String, ServiceInstance<Payload>>();
- private JsonSerDeser<CuratorServiceInstance<Payload>> deser =
- new JsonSerDeser<CuratorServiceInstance<Payload>>(CuratorServiceInstance.class);
+ private JsonSerDeser<CuratorServiceInstance> deser =
+ new JsonSerDeser<CuratorServiceInstance>(CuratorServiceInstance.class);
/**
* Create an instance
[45/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry (patch SLIDER-408)
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry (patch SLIDER-408)
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02623b1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02623b1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02623b1d
Branch: refs/heads/develop
Commit: 02623b1d73058cfb081098f712d699a1d57bc519
Parents: cc2a975 e375726
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 11:45:43 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 11:45:43 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 97 --------------------
.../slider/common/params/ActionGetConfArgs.java | 51 ----------
.../apache/slider/common/params/ClientArgs.java | 9 --
.../slider/common/params/SliderActions.java | 1 -
.../slider/client/TestCommonArgParsing.groovy | 31 -------
.../funtest/framework/CommandTestBase.groovy | 13 ---
.../commands/UnknownClusterOperationsIT.groovy | 6 --
.../lifecycle/AgentClusterLifecycleIT.groovy | 2 -
.../funtest/HBaseClusterLifecycleIT.groovy | 5 +-
9 files changed, 4 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02623b1d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index c5022ee,8d2bcfd..6613117
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -70,10 -58,8 +70,9 @@@ import org.apache.slider.common.params.
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
- import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.Arguments;
@@@ -339,15 -319,11 +338,13 @@@ public class SliderClient extends Abstr
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
+ } else if (ACTION_DIAGNOSTIC.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
- } else if (ACTION_GETCONF.equals(action)) {
- exitCode = actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
} else if (ACTION_HELP.equals(action) ||
ACTION_USAGE.equals(action)) {
log.info(serviceArgs.usage());
@@@ -1891,101 -1925,10 +1888,7 @@@
return EXIT_SUCCESS;
}
- /*
- * Creates a site conf with entries from clientProperties of ClusterStatus
- * @param desc ClusterDescription, can be null
- * @param clustername, can be null
- * @return site conf
- */
- public Configuration getSiteConf(ClusterDescription desc, String clustername)
- throws YarnException, IOException {
- if (desc == null) {
- desc = getClusterDescription();
- }
- if (clustername == null) {
- clustername = getDeployedClusterName();
- }
- String description = "Slider Application Instance " + clustername;
-
- Configuration siteConf = new Configuration(false);
- for (String key : desc.clientProperties.keySet()) {
- siteConf.set(key, desc.clientProperties.get(key), description);
- }
- return siteConf;
- }
--
-
-- /**
- * get the cluster configuration
- * @param clustername cluster name
- * @return the cluster name
- */
-
- @SuppressWarnings(
- {"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs)
- throws YarnException, IOException {
- File outfile = null;
-
- if (confArgs.getOutput() != null) {
- outfile = new File(confArgs.getOutput());
- }
-
- String format = confArgs.getFormat();
- verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
- ClusterDescription status = getClusterDescription(clustername);
- Writer writer;
- boolean toPrint;
- if (outfile != null) {
- writer = new FileWriter(outfile);
- toPrint = false;
- } else {
- writer = new StringWriter();
- toPrint = true;
- }
- try {
- String description = "Slider Application Instance " + clustername;
- // JDK7
- /*
- switch (format) {
- case Arguments.FORMAT_XML:
- Configuration siteConf = getSiteConf(status, clustername);
- siteConf.writeXml(writer);
- break;
- case Arguments.FORMAT_PROPERTIES:
- Properties props = new Properties();
- props.putAll(status.clientProperties);
- props.store(writer, description);
- break;
- default:
- throw new BadCommandArgumentsException("Unknown format: " + format);
- }
- */
- if (Arguments.FORMAT_XML.equals(format)) {
- Configuration siteConf = getSiteConf(status, clustername);
- siteConf.writeXml(writer);
- } else if (Arguments.FORMAT_PROPERTIES.equals(format)) {
- Properties props = new Properties();
- props.putAll(status.clientProperties);
- props.store(writer, description);
- } else {
- throw new BadCommandArgumentsException("Unknown format: " + format);
- }
- } finally {
- // data is written.
- // close the file
- writer.close();
- }
- // then, if this is not a file write, print it
- if (toPrint) {
- // not logged
- System.err.println(writer.toString());
- }
- return EXIT_SUCCESS;
- }
-
- * Restore a cluster
- */
+ @Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
SliderUtils.validateClusterName(clustername);
// see if it is actually running and bail out;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02623b1d/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 16363d2,50bf106..a4265aa
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@@ -78,26 -76,24 +77,25 @@@ public class ClientArgs extends CommonA
protected void addActionArguments() {
addActions(
- actionAMSuicideArgs,
- actionBuildArgs,
- actionCreateArgs,
- actionUpdateArgs,
- actionDestroyArgs,
- actionExistsArgs,
- actionFlexArgs,
- actionFreezeArgs,
- actionKillContainerArgs,
- actionListArgs,
- actionRegistryArgs,
- actionStatusArgs,
- actionThawArgs,
- actionHelpArgs,
- actionVersionArgs,
- actionInstallPackageArgs,
- actionDiagnosticArgs
- );
+ actionAMSuicideArgs,
+ actionBuildArgs,
+ actionCreateArgs,
+ actionUpdateArgs,
+ actionDestroyArgs,
+ actionDiagnosticArgs,
+ actionExistsArgs,
+ actionFlexArgs,
+ actionFreezeArgs,
- actionGetConfArgs,
+ actionHelpArgs,
+ actionInstallPackageArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionRegistryArgs,
+ actionResolveArgs,
+ actionStatusArgs,
+ actionThawArgs,
+ actionVersionArgs
+ );
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02623b1d/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
[26/50] [abbrv] git commit: SLIDER-149 purged all references/uses of
the curator service discovery and the original slider registry code.
Posted by st...@apache.org.
SLIDER-149 purged all references/uses of the curator service discovery and the original slider registry code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2154c753
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2154c753
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2154c753
Branch: refs/heads/develop
Commit: 2154c753023fb9f3372945277f5a63905a5b0c6d
Parents: da814b7
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 25 21:15:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 25 21:15:36 2014 +0100
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +-
slider-core/pom.xml | 10 -
.../org/apache/slider/client/SliderClient.java | 22 --
.../slider/core/registry/info/RegistryView.java | 42 ---
.../core/registry/info/ServiceInstanceData.java | 140 ---------
.../slideram/SliderAMClientProvider.java | 6 -
.../server/appmaster/web/SliderAMWebApp.java | 8 -
.../server/services/curator/CuratorHelper.java | 133 --------
.../server/services/curator/CuratorService.java | 115 -------
.../curator/CuratorServiceInstance.java | 78 -----
.../curator/CuratorServiceInstances.java | 42 ---
.../server/services/curator/CuratorUriSpec.java | 58 ----
.../services/curator/RegistryBinderService.java | 315 -------------------
.../registry/RegistryDiscoveryContext.java | 37 ---
.../registry/RegistryRestResources.java | 179 -----------
.../registry/RegistryServiceConstants.java | 27 --
.../registry/SliderRegistryService.java | 95 ------
.../YarnRegistryViewForProviders.java | 30 +-
.../standalone/TestStandaloneAgentAM.groovy | 4 -
.../apache/slider/test/SliderTestUtils.groovy | 5 -
.../slider/providers/hbase/HBaseKeys.java | 2 +-
.../providers/hbase/HBaseProviderService.java | 33 +-
22 files changed, 47 insertions(+), 1337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 613769e..5d4468c 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -34,10 +34,9 @@ import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.persist.ConfTreeSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfiguration
-import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.server.services.curator.CuratorServiceInstance
+
import org.junit.Before
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 8263199..d570606 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -346,16 +346,6 @@
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-x-discovery</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-x-discovery-server</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index de27dcd..63bf9c2 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -109,8 +109,6 @@ import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.core.registry.retrieve.RegistryRetriever;
import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
@@ -2472,26 +2470,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- private void logInstance(ServiceInstanceData instance,
- boolean verbose) {
- if (!verbose) {
- log.info("{}", instance.id);
- } else {
- log.info("{}: ", instance.id);
- logEndpoints(instance);
- }
- }
-
- private void logEndpoints(ServiceInstanceData instance) {
- Map<String, RegisteredEndpoint> endpoints =
- instance.listEndpoints(true);
- for (Map.Entry<String, RegisteredEndpoint> entry : endpoints.entrySet()) {
- String name = entry.getKey();
- RegisteredEndpoint endpoint = entry.getValue();
- log.info(" {}", endpoint);
- }
- }
-
/**
* list configs available for an instance
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
deleted file mode 100644
index 89b9c6e..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Deprecated
-public class RegistryView {
-
- /**
- * Endpoints
- */
- public Map<String, RegisteredEndpoint> endpoints =
- new HashMap<String, RegisteredEndpoint>(2);
-
- public String configurationsURL;
-
- public String documentsURL;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
deleted file mode 100644
index 52ae2e2..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Service instance data to serialize with JSON.
- *
- * The equality and hash codes are derived from the
- * service type and ID, which aren't final so that JSON marshalling
- * works. Do not change these fields if an instance is stored
- * in a map
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Deprecated
-public class ServiceInstanceData implements Serializable {
-
- public String serviceType;
- public String id;
- public String description;
- public String yarnApplicationId;
- public long registrationTimeUTC;
-
- /**
- * Anything can go into the internal view, it's specific
- * to the provider
- */
- public RegistryView internalView = new RegistryView();
-
- /**
- * External view of the service: public information
- */
- public RegistryView externalView = new RegistryView();
-
- public ServiceInstanceData() {
- }
-
- public ServiceInstanceData(String id, String serviceType) {
- this.serviceType = serviceType;
- this.id = id;
- }
-
- /**
- * Instances are equal if they look after the same service type
- * and name
- * @param o other
- * @return true if id and type match
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ServiceInstanceData that = (ServiceInstanceData) o;
-
- if (!id.equals(that.id)) {
- return false;
- }
- if (!serviceType.equals(that.serviceType)) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = serviceType.hashCode();
- result = 31 * result + id.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ServiceInstanceData{");
- sb.append("id='").append(id).append('\'');
- sb.append(", serviceType='").append(serviceType).append('\'');
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * get the internal or external registry
- * @param external flag to indicate the external endpoints
- * @return a view -which may be null
- */
- public RegistryView getRegistryView(boolean external) {
- return external ? externalView : internalView;
- }
-
- /**
- * List the internal or external endpoints. This returns
- * an empty list if there are none registered
- * @param external flag to indicate the external endpoints
- * @return a map of published endpoints
- */
- public Map<String, RegisteredEndpoint> listEndpoints(boolean external) {
- RegistryView view = getRegistryView(external);
- if (view == null) {
- return new HashMap<String, RegisteredEndpoint>(0);
- }
- Map<String, RegisteredEndpoint> endpoints = view.endpoints;
- if (endpoints != null) {
- return endpoints;
- } else {
- return new HashMap<String, RegisteredEndpoint>(0);
- }
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index dc84f02..68562e4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -23,8 +23,6 @@ import com.codahale.metrics.MetricRegistry;
import com.google.gson.GsonBuilder;
import org.apache.curator.CuratorZookeeperClient;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.server.entity.ServiceNames;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -165,8 +163,6 @@ public class SliderAMClientProvider extends AbstractClientProvider
CuratorFramework.class,
CuratorZookeeperClient.class,
- ServiceInstance.class,
- ServiceNames.class,
MetricRegistry.class
};
String[] jars =
@@ -177,8 +173,6 @@ public class SliderAMClientProvider extends AbstractClientProvider
"curator-framework.jar",
"curator-client.jar",
- "curator-x-discovery.jar",
- "curator-x-discovery-service.jar",
"metrics-core.jar"
};
ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 64be603..34080d6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -21,21 +21,13 @@ import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.curator.x.discovery.ServiceDiscovery;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
import org.apache.hadoop.yarn.webapp.Dispatcher;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
-import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
-import org.apache.slider.server.services.curator.CuratorHelper;
-import org.apache.slider.server.services.registry.RegistryDiscoveryContext;
-import org.apache.slider.server.services.registry.RegistryRestResources;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import java.util.HashMap;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
deleted file mode 100644
index 96d296d..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
-import org.apache.curator.x.discovery.strategies.RandomStrategy;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.services.registry.RegistryServiceConstants;
-import org.apache.slider.server.services.registry.RegistryDiscoveryContext;
-import org.apache.slider.server.services.registry.SliderRegistryService;
-
-/**
- * This class creates a curator -but does not start or close it. That
- * is the responsbility of the owner
- */
-@Deprecated
-public class CuratorHelper extends Configured {
-
- private final CuratorFramework curator;
- private final String connectionString;
-
- public CuratorHelper(Configuration conf, String connectionString) {
- super(conf);
- this.connectionString = connectionString;
- curator = createCurator(this.connectionString);
- }
-
-
- public CuratorFramework getCurator() {
- return curator;
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- /**
- * Create a retry policy for this configuration
- * @param conf
- * @return
- */
- public RetryPolicy createRetryPolicy() {
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- return retryPolicy;
- }
-
- private CuratorFramework createCurator(String connectionString) {
- CuratorFramework curator =
- CuratorFrameworkFactory.newClient(connectionString,
- createRetryPolicy());
- return curator;
- }
-
- /**
- * Create a discovery builder bonded to this curator
- * @return
- */
- public ServiceDiscoveryBuilder<ServiceInstanceData> createDiscoveryBuilder() {
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
- ServiceDiscoveryBuilder.builder(ServiceInstanceData.class);
- discoveryBuilder.client(curator);
- return discoveryBuilder;
- }
-
- /**
- * Create an instance
- * @param discoveryBuilder builder to create the discovery from
- */
-
- public SliderRegistryService createRegistryBinderService(
- String basePath,
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder) {
- discoveryBuilder.basePath(basePath);
- return new SliderRegistryService(curator,
- basePath,
- discoveryBuilder.build());
- }
-
-
- /**
- * Create an instance -including the initial binder
- * @param basePath base path
- * @return the binder service
- */
- public SliderRegistryService createRegistryBinderService(
- String basePath) {
- ServiceDiscoveryBuilder<ServiceInstanceData> discoveryBuilder =
- createDiscoveryBuilder();
- //registry will start curator as well as the binder, in the correct order
- SliderRegistryService registryBinderService =
- createRegistryBinderService(basePath, discoveryBuilder);
- return registryBinderService;
- }
-
- public RegistryDiscoveryContext createDiscoveryContext(
- ServiceDiscovery<ServiceInstanceData> discovery) {
- Preconditions.checkNotNull(discovery);
- return new RegistryDiscoveryContext(discovery,
- new RandomStrategy<ServiceInstanceData>(),
- RegistryServiceConstants.INSTANCE_REFRESH_MS,
- ServiceInstanceData.class);
-
- }
-
- @Override
- public String toString() {
- return "Curator binding " + connectionString;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
deleted file mode 100644
index a98a192..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.utils.ZKPaths;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.registry.client.binding.ZKPathDumper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-
-@Deprecated
-public class CuratorService extends AbstractService {
- private static final Logger log =
- LoggerFactory.getLogger(CuratorService.class);
- protected final String basePath;
-
- private final CuratorFramework curator;
- private CuratorHelper curatorHelper;
-
-
- public CuratorService(String name,
- CuratorFramework curator,
- String basePath) {
- super(name);
- this.curator = Preconditions.checkNotNull(curator, "null curator");
- this.basePath = basePath;
- }
-
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- curatorHelper = new CuratorHelper(conf, basePath);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- curator.start();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- closeCuratorComponent(curator);
- }
-
- public final CuratorFramework getCurator() {
- return curator;
- }
-
- protected final void closeCuratorComponent(Closeable closeable) {
- try {
- IOUtils.closeStream(closeable);
- } catch (Throwable ignored) {
- //triggered on an attempt to close before started
- log.debug("Error when closing {}", ignored);
- }
- }
-
- public final String pathForServicetype(String servicetype) {
- return ZKPaths.makePath(getBasePath(), servicetype);
- }
-
- protected final String pathForInstance(String servicetype, String id) {
- Preconditions.checkNotNull(servicetype);
- Preconditions.checkNotNull(id);
- return ZKPaths.makePath(pathForServicetype(servicetype), id);
- }
-
- public final String getBasePath() {
- return basePath;
- }
-
- public final CuratorHelper getCuratorHelper() {
- return curatorHelper;
- }
-
- @Override
- public String toString() {
- return super.toString() + "; "
- + (curatorHelper != null ? curatorHelper : "( unbound)")
- + "; " + basePath;
- }
-
- /**
- * Get an on-demand path jumper
- * @return a class that can dump the contents of the registry
- */
- public ZKPathDumper dumpPath(boolean verbose) {
- return new ZKPathDumper(curator, basePath);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
deleted file mode 100644
index 0acadb2..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.ServiceType;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonTypeInfo;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@Deprecated
-public class CuratorServiceInstance<T> {
-
- public String name;
- public String id;
- public String address;
- public Integer port;
- public Integer sslPort;
- public T payload;
- public long registrationTimeUTC;
- public ServiceType serviceType;
- public CuratorUriSpec uriSpec;
-
- public CuratorServiceInstance() {
- }
-
- public CuratorServiceInstance(ServiceInstance<T> si) {
- this.name = si.getName();
- this.id = si.getId();
- this.address = si.getAddress();
- this.port = si.getPort();
- this.sslPort = si.getSslPort();
- this.payload = si.getPayload();
- this.registrationTimeUTC = si.getRegistrationTimeUTC();
- this.serviceType = si.getServiceType();
- this.uriSpec = new CuratorUriSpec();
- this.uriSpec.addAll(si.getUriSpec().getParts());
- }
-
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CuratorServiceInstance{");
- sb.append("name='").append(name).append('\'');
- sb.append(", id='").append(id).append('\'');
- sb.append(", address='").append(address).append('\'');
- sb.append(", port=").append(port);
- sb.append(", sslPort=").append(sslPort);
- sb.append(", payload=").append(payload);
- sb.append(", registrationTimeUTC=").append(registrationTimeUTC);
- sb.append(", serviceType=").append(serviceType);
- sb.append(", uriSpec=").append(uriSpec);
- sb.append('}');
- return sb.toString();
- }
-
- @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS)
- public T getPayload() {
- return payload;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
deleted file mode 100644
index 2b575be..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.curator;
-
-import com.google.common.collect.Lists;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- *
- */
-@Deprecated
-public class CuratorServiceInstances<T> {
- private final List<CuratorServiceInstance<T>> services;
-
- public CuratorServiceInstances() {
- services = Lists.newArrayList();
- }
-
- public CuratorServiceInstances(Collection<? extends CuratorServiceInstance<T>> c) {
- services = Lists.newArrayList(c);
- }
-
- public List<CuratorServiceInstance<T>> getServices() {
- return Lists.newArrayList(services);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
deleted file mode 100644
index 478d3f4..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.curator;
-
-import com.google.common.collect.Lists;
-import org.apache.curator.x.discovery.UriSpec;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-
-import java.util.List;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@Deprecated
-public class CuratorUriSpec extends UriSpec{
-
- private final List<Part> parts = Lists.newArrayList();
-
- public CuratorUriSpec() {
- super();
- }
-
- @Override
- public List<Part> getParts() {
- return Lists.newArrayList(parts);
- }
-
- @Override
- public void add(Part part) {
- super.add(part);
- parts.add(part);
- }
-
- public void addAll(List<Part> parts) {
- parts.addAll(parts);
- }
-
- @Override
- public void remove(Part part) {
- super.remove(part);
- parts.remove(part);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
deleted file mode 100644
index fca967b..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.curator;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.ServiceInstanceBuilder;
-import org.apache.curator.x.discovery.ServiceType;
-import org.apache.curator.x.discovery.UriSpec;
-import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * YARN service for Curator service discovery; the discovery instance's
- * start/close methods are tied to the lifecycle of this service
- * @param <Payload> the payload of the operation
- */
-@Deprecated
-public class RegistryBinderService<Payload> extends CuratorService {
- private static final Logger log =
- LoggerFactory.getLogger(RegistryBinderService.class);
-
- private final ServiceDiscovery<Payload> discovery;
-
- private final Map<String, ServiceInstance<Payload>> entries =
- new HashMap<String, ServiceInstance<Payload>>();
-
- private JsonSerDeser<CuratorServiceInstance> deser =
- new JsonSerDeser<CuratorServiceInstance>(CuratorServiceInstance.class);
-
- /**
- * Create an instance
- * @param curator Does not need to be started
- * @param basePath base directory
- * @param discovery discovery instance -not yet started
- */
- public RegistryBinderService(CuratorFramework curator,
- String basePath,
- ServiceDiscovery<Payload> discovery) {
- super("RegistryBinderService", curator, basePath);
-
- this.discovery =
- Preconditions.checkNotNull(discovery, "null discovery arg");
- }
-
-
- public ServiceDiscovery<Payload> getDiscovery() {
- return discovery;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- discovery.start();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- closeCuratorComponent(discovery);
- super.serviceStop();
- }
-
- /**
- * register an instance -only valid once the service is started
- * @param id ID -must be unique
- * @param name name
- * @param url URL
- * @param payload payload (may be null)
- * @return the instance
- * @throws Exception on registration problems
- */
- public ServiceInstance<Payload> register(String name,
- String id,
- URL url,
- Payload payload) throws Exception {
- Preconditions.checkNotNull(id, "null `id` arg");
- Preconditions.checkNotNull(name, "null `name` arg");
- Preconditions.checkState(isInState(STATE.STARTED), "Not started: " + this);
-
- ServiceInstanceBuilder<Payload> instanceBuilder = builder()
- .name(name)
- .id(id)
- .payload(payload)
- .serviceType(ServiceType.DYNAMIC);
- if (url != null) {
- UriSpec uriSpec = new UriSpec(url.toString());
-
- int port = url.getPort();
- if (port == 0) {
- throw new IOException("Port undefined in " + url);
- }
- instanceBuilder
- .uriSpec(uriSpec)
- .port(port);
- }
- ServiceInstance<Payload> instance = instanceBuilder.build();
- log.info("registering {}", instance.toString());
- discovery.registerService(instance);
- log.info("registration completed {}", instance.toString());
- synchronized (this) {
- entries.put(id, instance);
- }
- return instance;
- }
-
- /**
- * Create a builder. This is already pre-prepared with address, registration
- * time and a (random) UUID
- * @return a builder
- * @throws IOException IO problems, including enumerating network ports
- */
- public ServiceInstanceBuilder<Payload> builder() throws
- IOException {
- try {
- return ServiceInstance.builder();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-
- /**
- * List all instance IDs of a service type
- * @param servicetype service type
- * @return list of matches
- * @throws Exception
- */
- public List<String> instanceIDs(String servicetype) throws Exception {
- Preconditions.checkNotNull(servicetype);
- List<String> instanceIds;
- try {
- instanceIds =
- getCurator().getChildren().forPath(pathForServicetype(servicetype));
- } catch (KeeperException.NoNodeException e) {
- instanceIds = Lists.newArrayList();
- }
- return instanceIds;
- }
-
- /**
- * List all service types registered
- * @return a list of service types
- * @throws Exception
- */
- public List<String> serviceTypes() throws Exception {
- List<String> types;
- try {
- types =
- getCurator().getChildren().forPath(getBasePath());
- } catch (KeeperException.NoNodeException e) {
- types = Lists.newArrayList();
- }
- return types;
- }
-
-
- /**
- * Return a service instance POJO
- *
- * @param servicetype name of the service
- * @param id ID of the instance
- * @return the instance or <code>null</code> if not found
- * @throws Exception errors
- */
- public CuratorServiceInstance<Payload> queryForInstance(String servicetype, String id)
- throws Exception {
- CuratorServiceInstance<Payload> instance = null;
- String path = pathForInstance(servicetype, id);
- try {
- byte[] bytes = getCurator().getData().forPath(path);
- if (bytes!=null && bytes.length>0) {
- instance = deser.fromBytes(bytes);
- }
- } catch (KeeperException.NoNodeException ignore) {
- // ignore
- }
- return instance;
- }
-
- /**
- * List all the instances
- * @param servicetype name of the service
- * @return a list of instances and their payloads
- * @throws IOException any problem
- */
- public List<CuratorServiceInstance<Payload>> listInstances(String servicetype) throws
- IOException {
- try {
- List<String> instanceIDs = instanceIDs(servicetype);
- List<CuratorServiceInstance<Payload>> instances =
- new ArrayList<CuratorServiceInstance<Payload>>(instanceIDs.size());
- for (String instanceID : instanceIDs) {
- CuratorServiceInstance<Payload> instance =
- queryForInstance(servicetype, instanceID);
- if (instance != null) {
- instances.add(instance);
- }
- }
- return instances;
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Find an instance with a given ID
- * @param instances instances
- * @param name ID to look for
- * @return the discovered instance or null
- */
- public CuratorServiceInstance<Payload> findByID(List<CuratorServiceInstance<Payload>> instances, String name) {
- Preconditions.checkNotNull(name);
- for (CuratorServiceInstance<Payload> instance : instances) {
- if (instance.id.equals(name)) {
- return instance;
- }
- }
- return null;
- }
-
- /**
- * Find a single instance -return that value or raise an exception
- * @param serviceType service type
- * @param name the name (required(
- * @return the instance that matches the criteria
- * @throws FileNotFoundException if there were no matches
- * @throws IOException any network problem
- */
- public CuratorServiceInstance<Payload> findInstance(String serviceType,
- String name) throws IOException {
- Preconditions.checkArgument(StringUtils.isNotEmpty(name), "name");
- return findInstances(serviceType, name).get(0);
- }
- /**
- * List registry entries. If a name was given, then the single match is returned
- * -otherwise all entries matching the service type
- * @param serviceType service type
- * @param name an optional name
- * @return the (non-empty) list of instances that match the criteria
- * @throws FileNotFoundException if there were no matches
- * @throws IOException any network problem
- */
- public List<CuratorServiceInstance<Payload>> findInstances(String serviceType,
- String name)
- throws FileNotFoundException, IOException {
- List<CuratorServiceInstance<Payload>> instances =
- listInstances(serviceType);
- if (instances.isEmpty()) {
- throw new FileNotFoundException(
- "No registry entries for service type " + serviceType);
- }
- if (StringUtils.isNotEmpty(name)) {
- CuratorServiceInstance<Payload> foundInstance = findByID(instances, name);
- if (foundInstance == null) {
- throw new FileNotFoundException(
- "No registry entries for service name " + name
- + " and service type " + serviceType);
- }
- instances.clear();
- instances.add(foundInstance);
- }
- return instances;
- }
-
- /**
- * Enum all service types in the registry
- * @return a possibly empty collection of service types
- * @throws IOException networking
- */
- public Collection<String> getServiceTypes() throws IOException {
- try {
- return getDiscovery().queryForNames();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
deleted file mode 100644
index d6b676f..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import org.apache.curator.x.discovery.ProviderStrategy;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.curator.x.discovery.server.contexts.GenericDiscoveryContext;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-
-@Deprecated
-public class RegistryDiscoveryContext extends GenericDiscoveryContext<ServiceInstanceData> {
-
- public RegistryDiscoveryContext(ServiceDiscovery<ServiceInstanceData> serviceDiscovery,
- ProviderStrategy<ServiceInstanceData> providerStrategy,
- int instanceRefreshMs,
- Class<ServiceInstanceData> payloadType) {
- super(serviceDiscovery, providerStrategy, instanceRefreshMs, payloadType);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
deleted file mode 100644
index ea97847..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import com.google.inject.Singleton;
-import org.apache.curator.x.discovery.ServiceInstance;
-import org.apache.curator.x.discovery.server.rest.DiscoveryContext;
-import org.apache.curator.x.discovery.server.rest.DiscoveryResource;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.curator.CuratorServiceInstances;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.net.URI;
-import java.net.URL;
-import java.util.List;
-import java.util.Random;
-
-@Singleton
-@Path(RestPaths.SLIDER_PATH_REGISTRY)
-@Deprecated
-public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData> {
- public static final String SERVICE_NAME = RestPaths.REGISTRY_SERVICE +"/{name}";
- public static final String SERVICE_NAME_ID = SERVICE_NAME + "/{id}";
- protected static final Logger log =
- LoggerFactory.getLogger(RegistryRestResources.class);
- private final SliderRegistryService registry;
- private DiscoveryContext<ServiceInstanceData> context;
- private Random randomizer = new Random();
-
- public RegistryRestResources(@Context DiscoveryContext<ServiceInstanceData> context,
- SliderRegistryService registry) {
- super(context);
- this.context = context;
- this.registry = registry;
- }
-
-// @GET
- public Response getWadl (@Context HttpServletRequest request) {
- try {
- URI location = new URL(request.getScheme(),
- request.getServerName(),
- request.getServerPort(),
- "/application.wadl").toURI();
- return Response.temporaryRedirect(location).build();
- } catch (Exception e) {
- log.error("Error during redirect to WADL", e);
- throw new WebApplicationException(Response.serverError().build());
- }
- }
-
- @javax.ws.rs.GET
- @javax.ws.rs.Produces({MediaType.APPLICATION_JSON})
- public Response getAtRoot() {
- try {
- List<String>
- instances = registry.serviceTypes();
- return Response.ok(instances).build();
- } catch (Exception e) {
- log.error("Error during generation of response", e);
- return Response.serverError().build();
- }
- }
-
-
- @Override
- @javax.ws.rs.GET
- @javax.ws.rs.Path(SERVICE_NAME)
- @javax.ws.rs.Produces({MediaType.APPLICATION_JSON})
- public Response getAll(@PathParam("name") String name) {
- try {
- List<CuratorServiceInstance<ServiceInstanceData>>
- instances = registry.listInstances(name);
- return Response.ok(
- new CuratorServiceInstances<ServiceInstanceData>(instances)).build();
- } catch (Exception e) {
- log.error("Error during generation of response", e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @GET
- @Path(SERVICE_NAME_ID)
- @Produces(MediaType.APPLICATION_JSON)
- public Response get(@PathParam("name") String name,
- @PathParam("id") String id) {
- try {
- CuratorServiceInstance<ServiceInstanceData> instance = registry.queryForInstance(name, id);
- if ( instance == null )
- {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
- Response.ResponseBuilder builder = Response.ok(instance);
- return builder.build();
- } catch (Exception e) {
- log.error("Trying to get instance {} from service {}: {})",
- id,
- name,
- e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @GET
- @Path("v1/anyservice/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public Response getAny(@PathParam("name") String name) {
- try {
- List<CuratorServiceInstance<ServiceInstanceData>>
- instances = registry.listInstances(name);
- if (instances == null || instances.isEmpty()) {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
-
- CuratorServiceInstance<ServiceInstanceData> randomInstance =
- instances.get(randomizer.nextInt(instances.size()));
- if ( randomInstance == null )
- {
- return Response.status(Response.Status.NOT_FOUND).build();
- }
- return Response.ok(randomInstance).build();
- } catch (Exception e) {
- log.error(String.format("Trying to get any instance from service (%s)", name), e);
- return Response.serverError().build();
- }
- }
-
- @Override
- @PUT
- @Path(SERVICE_NAME_ID)
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public Response putService(ServiceInstance<ServiceInstanceData> instance,
- @PathParam("name") String name,
- @PathParam("id") String id) {
- throw new UnsupportedOperationException("putService not supported");
- }
-
- @Override
- @DELETE
- @Path(SERVICE_NAME_ID)
- public Response removeService(@PathParam("name") String name,
- @PathParam("id") String id) {
- throw new UnsupportedOperationException("removeService not supported");
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
deleted file mode 100644
index f3a107f..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-/**
- * These constants are unique to the slider registry service itself
- */
-@Deprecated
-public class RegistryServiceConstants {
- public static final int INSTANCE_REFRESH_MS = 1000;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
deleted file mode 100644
index ab92c82..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.x.discovery.ServiceDiscovery;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.curator.RegistryBinderService;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * This is the registry service, which tries to hide exactly how the
- * registry is implemented
- */
-@Deprecated
-public class SliderRegistryService
- extends RegistryBinderService<ServiceInstanceData> {
-
- private ServiceInstanceData selfRegistration;
-
- public SliderRegistryService(CuratorFramework curator,
- String basePath,
- ServiceDiscovery<ServiceInstanceData> discovery) {
- super(curator, basePath, discovery);
- }
-
-
- public List<ServiceInstanceData> listInstancesByType(String serviceType) throws
- IOException {
- List<CuratorServiceInstance<ServiceInstanceData>> services =
- listInstances(serviceType);
- List<ServiceInstanceData> payloads = new ArrayList<ServiceInstanceData>(services.size());
- for (CuratorServiceInstance<ServiceInstanceData> instance : services) {
- payloads.add(instance.payload);
- }
- return payloads;
- }
-
- public ServiceInstanceData getSelfRegistration() {
- return selfRegistration;
- }
-
- private void setSelfRegistration(ServiceInstanceData selfRegistration) {
- this.selfRegistration = selfRegistration;
- }
-
- /**
- * register an instance -only valid once the service is started.
- * This sets the selfRegistration field
- * @param instanceData instance data
- * @param url URL to register
- * @throws IOException on registration problems
- */
- public void registerSelf(ServiceInstanceData instanceData, URL url) throws IOException {
- registerServiceInstance(instanceData, url);
- setSelfRegistration(instanceData);
- }
-
- public void registerServiceInstance(
- ServiceInstanceData instanceData, URL url) throws IOException {
- Preconditions.checkNotNull(instanceData);
- Preconditions.checkNotNull(instanceData.id);
- Preconditions.checkNotNull(instanceData.serviceType);
-
- try {
- register(instanceData.serviceType, instanceData.id, url, instanceData);
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index a08cfaf..d699279 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -102,18 +102,17 @@ public class YarnRegistryViewForProviders {
/**
- * Add a service under
- * @param componentName
- * @param record
- * @param ephemeral
+ * Add a service under a path
+ * @param username user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
* @throws IOException
*/
public void putService(String username,
String serviceClass,
String serviceName,
ServiceRecord record) throws IOException {
-
-
String path = RegistryOperationUtils.servicePath(
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
@@ -122,6 +121,25 @@ public class YarnRegistryViewForProviders {
}
+ /**
+ * Add a service under a path for the current user
+ * @param serviceClass service class to use under ~user
+ * @param serviceName name of the service
+ * @param record service record
+ * @throws IOException
+ */
+ public void putService(
+ String serviceClass,
+ String serviceName,
+ ServiceRecord record) throws IOException {
+ String path = RegistryOperationUtils.servicePath(
+ user, serviceClass, serviceName);
+ registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
+ registryOperations.create(path, record, CreateFlags.OVERWRITE);
+
+ }
+
+
public void rmComponent(String componentName) throws IOException {
String path = RegistryOperationUtils.componentPath(
user, sliderServiceclass, instanceName,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index e04f9cb..941ca83 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -26,15 +26,11 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.ClusterNode
import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.ActionRegistryArgs
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
-import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
@CompileStatic
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 7621a4d..9f8e850 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -45,15 +45,10 @@ import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.exceptions.WaitTimeoutException
import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.info.ServiceInstanceData
-import org.apache.slider.server.services.curator.CuratorServiceInstance
import org.junit.Assert
import org.junit.Assume
-import java.lang.reflect.Array
-
import static Arguments.ARG_OPTION
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index 2a20438..064414e 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -88,7 +88,7 @@ public interface HBaseKeys {
/**
* Service type used in registry
*/
- String HBASE_SERVICE_TYPE = "org.apache.hbase";
+ String HBASE_SERVICE_TYPE = "org-apache-hbase";
String HBASE_SITE_PUBLISHED_CONFIG = "hbase-site";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2154c753/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 372038b..3c059d9 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -18,10 +18,10 @@
package org.apache.slider.providers.hbase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
@@ -37,7 +37,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
@@ -57,6 +57,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
@@ -253,21 +254,29 @@ public class HBaseProviderService extends AbstractProviderService
private void registerHBaseServiceEntry() throws IOException {
- String name = amState.getApplicationName() ;
-// name += ".hbase";
- ServiceInstanceData instanceData = new ServiceInstanceData(name,
- HBASE_SERVICE_TYPE);
- log.info("registering {}/{}", name, HBASE_SERVICE_TYPE );
+
+ String name = amState.getApplicationName() ;
+ ServiceRecord serviceRecord = new ServiceRecord();
+
+ try {
+ URL configURL = new URL(amWebAPI, SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
+
+ serviceRecord.addExternalEndpoint(
+ RegistryTypeUtils.restEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API,
+ configURL.toURI()));
+ } catch (URISyntaxException e) {
+ log.warn("failed to create config URL: {}", e, e);
+ }
+ log.info("registering {}/{}", name, HBASE_SERVICE_TYPE);
+ yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord);
+
PublishedConfiguration publishedSite =
new PublishedConfiguration("HBase site", siteConf);
PublishedConfigSet configSet =
amState.getOrCreatePublishedConfigSet(HBASE_SERVICE_TYPE);
- instanceData.externalView.configurationsURL = SliderUtils.appendToURL(
- amWebAPI.toExternalForm(), SLIDER_PATH_PUBLISHER, HBASE_SERVICE_TYPE);
- configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite);
- registry.registerServiceInstance(instanceData, null);
-
+ configSet.put(HBASE_SITE_PUBLISHED_CONFIG, publishedSite);
}
/**
[49/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/851f4474
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/851f4474
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/851f4474
Branch: refs/heads/develop
Commit: 851f447484cd291eec21bb576f75ba0ad6f60565
Parents: 66957d4 3b32b8f
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 15:51:53 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 15:51:53 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/metainfo.xml | 2 -
app-packages/hbase/metainfo.xml | 4 -
app-packages/hbase/pom.xml | 1 +
app-packages/storm/metainfo.xml | 3 +
.../org/apache/slider/client/SliderClient.java | 20 +++--
.../providers/AbstractClientProvider.java | 4 +-
.../providers/agent/AgentClientProvider.java | 79 +++++++++++++++-----
.../slider/providers/agent/AgentKeys.java | 2 +-
.../providers/agent/AgentProviderService.java | 43 ++++++++---
.../agent/application/metadata/Component.java | 28 +++++++
.../agent/application/metadata/Metainfo.java | 19 +++++
.../slideram/SliderAMClientProvider.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 2 +-
.../providers/agent/TestBuildBasicAgent.groovy | 65 ++++++++++++++--
.../agent/TestAgentClientProvider.java | 2 +-
.../publisher/TestAgentProviderService.java | 1 +
slider-core/src/test/python/metainfo.xml | 21 ++++++
.../providers/agent/tests/bad/resources-3.json | 13 ++++
.../providers/agent/tests/good/resources.json | 4 +-
.../accumulo/AccumuloClientProvider.java | 4 +-
.../accumulo/AccumuloProviderService.java | 2 +-
.../providers/hbase/HBaseClientProvider.java | 4 +-
.../providers/hbase/HBaseProviderService.java | 4 +-
23 files changed, 264 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a920a3e,31a9a28..6a3f48b
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@@ -815,9 -809,11 +815,9 @@@ public class SliderAppMaster extends Ab
appState.noteAMLaunched();
-- //Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, actionQueues,
- liveContainers);
- sliderAMProvider.bind(stateForProviders, registry, actionQueues,
- liveContainers);
++ //Give the provider access to the state, and AM
+ providerService.bind(stateForProviders, actionQueues, liveContainers);
+ sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers);
// chaos monkey
maybeStartMonkey();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/851f4474/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
[36/50] [abbrv] git commit: SLIDER-149 in sync with latest YARN-913
changes
Posted by st...@apache.org.
SLIDER-149 in sync with latest YARN-913 changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27381a6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27381a6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27381a6d
Branch: refs/heads/develop
Commit: 27381a6dc29589a06d98be43901f175230d54198
Parents: d084e9e
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 6 21:20:02 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 6 21:20:02 2014 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 10 ++-
.../org/apache/slider/client/SliderClient.java | 64 ++++++++++++--------
.../server/appmaster/SliderAppMaster.java | 18 +++---
.../YarnRegistryViewForProviders.java | 8 +--
.../TestStandaloneYarnRegistryAM.groovy | 19 +++---
.../registry/TestRegistryRestMarshalling.groovy | 5 +-
.../registry/TestRegistryRestResources.groovy | 8 ++-
.../apache/slider/test/MicroZKCluster.groovy | 2 +-
.../providers/hbase/HBaseProviderService.java | 9 ++-
.../minicluster/live/TestHBaseMaster.groovy | 3 +-
10 files changed, 86 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index fd8af9a..2f6674e 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -30,7 +30,6 @@ import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.persist.ConfTreeSerDeser
import org.apache.slider.core.registry.docstore.PublishedConfiguration
@@ -40,6 +39,9 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.currentUser
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.servicePath
+
@Slf4j
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
protected static final String PROVIDER_PROPERTY = "site.accumulo-site." +
@@ -170,8 +172,10 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
Exception caught;
while (true) {
try {
- ServiceRecord instance =
- sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName)
+ String path = servicePath(currentUser(),
+ SliderKeys.APP_TYPE,
+ clusterName);
+ ServiceRecord instance = sliderClient.resolve(path)
RegistryRetriever retriever = new RegistryRetriever(instance)
PublishedConfiguration configuration = retriever.retrieveConfiguration(
retriever.getConfigurations(true), "quicklinks", true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index a5bbccc..602d6cf 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -48,7 +48,6 @@ import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*;
import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -127,15 +126,11 @@ import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
-import org.codehaus.jackson.map.DeserializationConfig;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.SerializationConfig;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
@@ -2402,15 +2397,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// the arguments
args.validate();
RegistryOperations operations = getRegistryOperations();
- String serviceclassPath = args.path;
+ String path = args.path;
Collection<ServiceRecord> serviceRecords;
try {
if (args.list) {
- actionRegistryListYarn(args);
+ Map<String, ServiceRecord> recordMap =
+ listServiceRecords(operations, path);
+ serviceRecords = recordMap.values();
+ // list records out
} else {
- ServiceRecord instance = lookupServiceRecord(registryArgs);
+ ServiceRecord instance = resolve(path);
serviceRecords = new ArrayList<ServiceRecord>(1);
serviceRecords.add(instance);
+ // list or save records
}
// JDK7
} catch (FileNotFoundException e) {
@@ -2442,14 +2441,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
registryArgs.validate();
try {
if (registryArgs.list) {
- actionRegistryListYarn(registryArgs);
+ actionRegistryList(registryArgs);
} else if (registryArgs.listConf) {
// list the configurations
actionRegistryListConfigsYarn(registryArgs);
} else if (SliderUtils.isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
- actionRegistryGetConfigYarn(registryArgs);
+ actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
} else {
// it's an unknown command
@@ -2478,7 +2477,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public Collection<ServiceRecord> actionRegistryListYarn(
+ public Collection<ServiceRecord> actionRegistryList(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
String serviceType = registryArgs.serviceType;
@@ -2486,23 +2485,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
RegistryOperations operations = getRegistryOperations();
Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
- String serviceclassPath =
+ String path =
serviceclassPath(
currentUser(),
serviceType);
try {
Map<String, ServiceRecord> recordMap =
- listServiceRecords(operations, serviceclassPath);
- RegistryPathStatus[] listDir;
+ listServiceRecords(operations, path);
if (recordMap.isEmpty()) {
throw new UnknownApplicationInstanceException(
- "No applications registered under " + serviceclassPath);
+ "No applications registered under " + path);
}
serviceRecords = recordMap.values();
} catch (PathNotFoundException e) {
- throw new UnknownApplicationInstanceException(e.getPath().toString(),
- e);
+ throw new UnknownApplicationInstanceException(path, e);
}
} else {
ServiceRecord instance = lookupServiceRecord(registryArgs);
@@ -2792,9 +2789,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.yarn_id);
+ log.info("{}", instance.getYarn_id());
} else {
- log.info("{}: ", instance.yarn_id);
+ log.info("{}: ", instance.getYarn_id());
logEndpoints(instance);
}
}
@@ -2848,7 +2845,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws FileNotFoundException if the config is not found
*/
@VisibleForTesting
- public PublishedConfiguration actionRegistryGetConfigYarn(ActionRegistryArgs registryArgs)
+ public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
ServiceRecord instance = lookupServiceRecord(registryArgs);
@@ -2910,7 +2907,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
SliderException,
IOException {
- return lookupServiceRecord(registryArgs.serviceType, registryArgs.name);
+ String path = servicePath(currentUser(), registryArgs.serviceType,
+ registryArgs.name);
+ return resolve(path);
}
/**
@@ -2925,18 +2924,33 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
public ServiceRecord lookupServiceRecord(String serviceType, String id)
throws IOException, SliderException {
+ String path = servicePath(currentUser(), serviceType, id);
+ return resolve(path);
+ }
+
+ /**
+ *
+ * Look up an instance
+ * @param path path
+ * @return instance data
+ * @throws UnknownApplicationInstanceException no path or service record
+ * at the end of the path
+ * @throws SliderException other failures
+ * @throws IOException IO problems or wrapped exceptions
+ */
+ public ServiceRecord resolve(String path)
+ throws IOException, SliderException {
try {
return getRegistryOperations().resolve(
- servicePath(currentUser(),
- serviceType, id));
+ path);
// TODO JDK7 SWITCH
} catch (PathNotFoundException e) {
throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
} catch (NoRecordException e) {
throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
}
- }
-
+ }
+
/**
* List instances in the registry for the current user
* @return a list of slider registry instances
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a259e42..6f7275c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
+import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
@@ -931,8 +931,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
- serviceRecord.yarn_id = appid.toString();
- serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION;
+ serviceRecord.putYarn_id(appid.toString());
+ serviceRecord.putYarn_persistence(PersistencePolicies.APPLICATION);
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -964,8 +964,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and a shorter lived binding to the app
String attempt = appAttemptID.toString();
ServiceRecord attemptRecord = new ServiceRecord(serviceRecord);
- attemptRecord.yarn_id = attempt;
- attemptRecord.yarn_persistence = PersistencePolicies.APPLICATION_ATTEMPT;
+ attemptRecord.putYarn_id(attempt);
+ attemptRecord.putYarn_persistence(PersistencePolicies.APPLICATION_ATTEMPT);
yarnRegistryOperations.putComponent(
RegistryPathUtils.encodeYarnID(attempt),
serviceRecord);
@@ -1007,10 +1007,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// this is where component registrations will go
log.info("Registering component {}", id);
String cid = RegistryPathUtils.encodeYarnID(id.toString());
- ServiceRecord container = new ServiceRecord(
- cid,
- description,
- PersistencePolicies.CONTAINER, null);
+ ServiceRecord container = new ServiceRecord();
+ container.putYarn_id(cid);
+ container.description = description;
+ container.putYarn_persistence(PersistencePolicies.CONTAINER);
try {
yarnRegistryOperations.putComponent(cid, container);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index d0c891d..10ca85c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -19,11 +19,11 @@
package org.apache.slider.server.services.yarnregistry;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.registry.client.api.BindFlags;
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.api.CreateFlags;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import java.io.IOException;
@@ -107,7 +107,7 @@ public class YarnRegistryViewForProviders {
String path = RegistryUtils.componentPath(
user, serviceClass, serviceName, componentName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.create(path, record, CreateFlags.OVERWRITE);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
}
/**
@@ -125,7 +125,7 @@ public class YarnRegistryViewForProviders {
String path = RegistryUtils.servicePath(
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.create(path, record, CreateFlags.OVERWRITE);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
}
/**
@@ -142,7 +142,7 @@ public class YarnRegistryViewForProviders {
String path = RegistryUtils.servicePath(
user, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.create(path, record, CreateFlags.OVERWRITE);
+ registryOperations.bind(path, record, BindFlags.OVERWRITE);
}
public void rmComponent(String componentName) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index b9238f4..7e74625 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -25,9 +25,10 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsClient
+import org.apache.hadoop.yarn.registry.client.impl.RegistryOperationsClient
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import static org.apache.hadoop.yarn.registry.client.binding.RegistryUtils.*
@@ -158,7 +159,8 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
def serviceRecord = serviceRecords[0]
log.info(serviceRecord.toString())
- assert serviceRecord.yarn_id != null;
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
+ assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
def externalEndpoints = serviceRecord.external;
assert externalEndpoints.size() > 0
@@ -299,7 +301,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
registryArgs.list = true;
registryArgs.name = "unknown"
try {
- client.actionRegistryListYarn(registryArgs)
+ client.actionRegistryList(registryArgs)
} catch (UnknownApplicationInstanceException expected) {
// expected
}
@@ -309,7 +311,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
registryArgs.name = null
registryArgs.serviceType = "org-apache-hadoop"
try {
- client.actionRegistryListYarn(registryArgs)
+ client.actionRegistryList(registryArgs)
} catch (UnknownApplicationInstanceException expected) {
// expected
}
@@ -324,9 +326,12 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
//now expect list to work
describe registryArgs.toString()
- def listedInstance = client.actionRegistryListYarn(registryArgs)
- assert listedInstance[0].yarn_id == serviceRecord.yarn_id
-
+ def listedInstance = client.actionRegistryList(registryArgs)
+
+ def resolvedRecord = listedInstance[0]
+ assert resolvedRecord[YarnRegistryAttributes.YARN_ID] == serviceRecord.getYarn_id()
+ assert resolvedRecord[YarnRegistryAttributes.YARN_PERSISTENCE] == serviceRecord.getYarn_persistence()
+
// listconf
registryArgs.list = false;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
index 44c8f05..d67dfeb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestMarshalling.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.web.rest.registry
import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
import org.junit.Test
/**
@@ -39,8 +40,8 @@ class TestRegistryRestMarshalling {
def serviceRecord = unmarshalled.service
assert serviceRecord
- assert serviceRecord.yarn_id !=null
- assert serviceRecord.yarn_persistence != 0
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
+ assert serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
index e6d31fd..ee678f9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/TestRegistryRestResources.groovy
@@ -26,6 +26,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
@@ -157,8 +158,9 @@ class TestRegistryRestResources extends AgentTestBase {
def unmarshalled = pem.fromJson(responseStr)
def r1 = unmarshalled.service
assert r1
- assert r1.yarn_id != null
- assert r1.yarn_persistence != 0
+ assert r1[YarnRegistryAttributes.YARN_ID] != null
+ assert r1[YarnRegistryAttributes.YARN_PERSISTENCE] != ""
+
// and via the web resource AP
jsonBuilder = webResource.type(MediaType.APPLICATION_JSON);
@@ -166,7 +168,7 @@ class TestRegistryRestResources extends AgentTestBase {
def serviceRecord = entryResource.service
assert serviceRecord != null;
- assert serviceRecord.yarn_id != null;
+ assert serviceRecord[YarnRegistryAttributes.YARN_ID] != null
def externalEndpoints = serviceRecord.external;
assert externalEndpoints.size() > 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
index 7f72490..c2d6a32 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/MicroZKCluster.groovy
@@ -22,7 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService
+import org.apache.hadoop.yarn.registry.client.impl.zk.RegistryOperationsService
import org.apache.hadoop.yarn.registry.server.services.MicroZookeeperService
import org.apache.slider.common.tools.SliderUtils
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 24abdba..ae9084b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
+import org.apache.hadoop.yarn.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
@@ -40,7 +40,6 @@ import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
@@ -258,9 +257,9 @@ public class HBaseProviderService extends AbstractProviderService
String name = amState.getApplicationName() ;
ServiceRecord serviceRecord = new ServiceRecord();
// bond lifespan to the application
- serviceRecord.yarn_id = yarnRegistry.getApplicationAttemptId()
- .getApplicationId().toString();
- serviceRecord.yarn_persistence = PersistencePolicies.APPLICATION;
+ serviceRecord.putYarn_id(yarnRegistry.getApplicationAttemptId()
+ .getApplicationId().toString());
+ serviceRecord.putYarn_persistence(PersistencePolicies.APPLICATION);
try {
URL configURL = new URL(amWebAPI,
SLIDER_PATH_PUBLISHER + "/" + HBASE_SERVICE_TYPE);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27381a6d/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
index 84db30f..b73432a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.registry.client.binding.RegistryUtils
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
@@ -93,7 +94,7 @@ class TestHBaseMaster extends HBaseMiniClusterTestBase {
assert hbaseInstances.size() == 1
ServiceRecord hbaseServiceData = hbaseInstances[0]
log.info "HBase service 0 == $hbaseServiceData"
- assert hbaseServiceData.yarn_id
+ assert hbaseServiceData[YarnRegistryAttributes.YARN_ID]
RegistryRetriever retriever = new RegistryRetriever(hbaseServiceData)
log.info retriever.toString()
[38/50] [abbrv] git commit: SLIDER-149 static sample.json resync'd
with current schema
Posted by st...@apache.org.
SLIDER-149 static sample.json resync'd with current schema
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29c98240
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29c98240
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29c98240
Branch: refs/heads/develop
Commit: 29c982403ea6ae3f0c070d5a9e24a27608127e80
Parents: 330579e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 08:56:48 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 08:56:48 2014 -0700
----------------------------------------------------------------------
.../slider/server/appmaster/web/rest/registry/sample.json | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/29c98240/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
index b2d59fd..4d2c6db 100644
--- a/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
+++ b/slider-core/src/test/resources/org/apache/slider/server/appmaster/web/rest/registry/sample.json
@@ -1,9 +1,8 @@
{
"nodes": ["/users/example/services/org-apache-slider/test-registry-rest-resources/components"], "service": {
- "registrationTime": 0,
"description": "Slider Application Master",
- "yarn_id": "application_1411664296263_0001",
- "yarn_persistence": 1,
+ "yarn:id": "application_1411664296263_0001",
+ "yarn:persistence": 1,
"external": [
{
"api": "org.apache.slider.appmaster",
[29/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a3e5b372
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a3e5b372
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a3e5b372
Branch: refs/heads/develop
Commit: a3e5b372cc257b1df65ad35565eb743e0775a7af
Parents: 65e7684 43f9a94
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 26 14:48:14 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 26 14:50:02 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/README.txt | 9 +-
app-packages/accumulo/resources.json | 3 +
.../accumulo/src/test/resources/resources.json | 3 +
app-packages/app-pkg-template/README.txt | 7 +-
.../app-pkg-template/appConfig-default.json | 21 ++++
app-packages/app-pkg-template/appConfig.json | 21 ----
.../app-pkg-template/resources-default.json | 16 +++
app-packages/app-pkg-template/resources.json | 16 ---
app-packages/hbase-win/README.txt | 8 +-
app-packages/hbase-win/appConfig-default.json | 38 +++++++
app-packages/hbase-win/appConfig.json | 38 -------
app-packages/hbase-win/resources-default.json | 24 ++++
app-packages/hbase-win/resources.json | 21 ----
app-packages/hbase-win/src/assembly/hbase.xml | 4 +-
app-packages/hbase/README.txt | 7 +-
app-packages/hbase/appConfig-default.json | 47 ++++++++
app-packages/hbase/appConfig.json | 47 --------
app-packages/hbase/pom.xml | 4 +-
app-packages/hbase/resources-default.json | 36 ++++++
app-packages/hbase/resources.json | 36 ------
app-packages/hbase/src/assembly/hbase.xml | 4 +-
.../src/test/resources/resources-default.json | 24 ++++
.../hbase/src/test/resources/resources.json | 21 ----
app-packages/memcached-win/README.txt | 7 +-
.../memcached-win/appConfig-default.json | 24 ++++
app-packages/memcached-win/appConfig.json | 24 ----
.../memcached-win/resources-default.json | 16 +++
app-packages/memcached-win/resources.json | 16 ---
app-packages/memcached/README.txt | 7 +-
app-packages/memcached/appConfig-default.json | 24 ++++
app-packages/memcached/appConfig.json | 24 ----
app-packages/memcached/resources-default.json | 16 +++
app-packages/memcached/resources.json | 16 ---
app-packages/storm/README.txt | 7 +-
app-packages/storm/appConfig-default.json | 46 ++++++++
app-packages/storm/appConfig.json | 46 --------
app-packages/storm/resources-default.json | 34 ++++++
app-packages/storm/resources.json | 31 -----
app-packages/storm/src/assembly/storm.xml | 4 +-
pom.xml | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 15 ++-
.../src/main/python/agent/ProcessHelper.py | 5 +-
slider-agent/src/main/python/agent/main.py | 13 ++-
.../agent/TestCustomServiceOrchestrator.py | 20 +++-
.../org/apache/slider/api/ResourceKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 8 +-
.../slider/core/launch/AbstractLauncher.java | 52 +++++++++
.../slider/core/launch/AppMasterLauncher.java | 4 +-
.../slider/providers/agent/AgentKeys.java | 2 +-
.../providers/agent/AgentProviderService.java | 6 +-
.../web/rest/agent/ExecutionCommand.java | 1 +
.../actions/TestActionInstallPackage.groovy | 6 +-
.../core/launch/TestAppMasterLauncher.java | 112 +++++++++++++++++++
.../agent/TestAgentProviderService.java | 2 +-
54 files changed, 624 insertions(+), 424 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e5b372/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index fed95a8,b31d476..bfc52bd
--- a/pom.xml
+++ b/pom.xml
@@@ -136,10 -136,10 +136,10 @@@
<!--
core artifacts
-->
- <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
- <accumulo.version>1.6.1-SNAPSHOT</accumulo.version>
+ <accumulo.version>1.6.1</accumulo.version>
<!--
artifact versions
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e5b372/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e5b372/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a3e5b372/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
[03/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/901a6738
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/901a6738
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/901a6738
Branch: refs/heads/develop
Commit: 901a67385d6c7bce6cf8e47c58590282985b684c
Parents: 6e883bf f14d32c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 14:17:04 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 14:17:04 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 7 +-
.../src/test/resources/appConfig_kerberos.json | 57 ++
.../test/resources/appConfig_monitor_ssl.json | 7 +-
.../src/test/resources/appConfig_ssl.json | 7 +-
app-packages/hbase/appConfig.json | 8 +-
app-packages/storm/README.txt | 19 +-
app-packages/storm/appConfig.json | 16 +-
app-packages/storm/configuration/storm-env.xml | 25 +
app-packages/storm/metainfo.xml | 26 +-
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 -
app-packages/storm/package/scripts/params.py | 15 +-
app-packages/storm/package/scripts/service.py | 8 +
.../storm/package/scripts/status_params.py | 4 +-
.../storm/package/templates/config.yaml.j2 | 9 -
.../storm/package/templates/storm_jaas.conf.j2 | 27 +-
app-packages/storm/pom.xml | 90 +++
app-packages/storm/resources.json | 10 +-
app-packages/storm/src/assembly/storm.xml | 68 +++
pom.xml | 42 +-
slider-agent/pom.xml | 4 +-
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 --
slider-agent/src/test/python/python-wrap.sh | 42 ++
slider-agent/src/test/python/unitTests.py | 36 +-
slider-core/pom.xml | 4 +-
.../org/apache/slider/api/InternalKeys.java | 11 +
.../org/apache/slider/api/ResourceKeys.java | 13 -
.../org/apache/slider/client/SliderClient.java | 33 +-
.../common/params/AbstractActionArgs.java | 11 +
.../slider/common/params/ActionHelpArgs.java | 14 +-
.../slider/common/params/ActionVersionArgs.java | 12 +-
.../apache/slider/common/tools/SliderUtils.java | 571 ++++++++++++++-----
.../slider/core/build/InstanceBuilder.java | 5 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 3 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 38 +-
.../server/appmaster/RoleLaunchService.java | 8 +-
.../server/appmaster/SliderAppMaster.java | 68 ++-
.../server/appmaster/actions/ActionHalt.java | 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +-
.../appmaster/actions/ActionStartContainer.java | 8 +-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 +-
.../actions/ProviderReportedContainerLoss.java | 13 +-
.../actions/ProviderStartupCompleted.java | 4 -
.../server/appmaster/actions/QueueAccess.java | 5 +
.../server/appmaster/actions/QueueService.java | 11 +-
.../actions/RegisterComponentInstance.java | 5 +
.../actions/ReviewAndFlexApplicationSize.java | 43 ++
.../actions/UnregisterComponentInstance.java | 7 +-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 11 +-
.../services/utility/EndOfServiceWaiter.java | 84 +++
.../services/workflow/ForkedProcessService.java | 21 +-
.../services/workflow/LongLivedProcess.java | 57 +-
.../agent/AgentMiniClusterTestBase.groovy | 9 +
.../agent/actions/TestActionVersion.groovy | 5 +-
.../standalone/TestStandaloneAMKill.groovy | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy | 8 +-
.../tools/TestClientResourceRegistration.groovy | 3 +
.../common/tools/TestConfigHelperHDFS.groovy | 3 +
.../tools/TestExecutionEnvironment.groovy | 42 ++
.../common/tools/TestMiscSliderUtils.groovy | 3 +
.../common/tools/TestWindowsSupport.groovy | 76 ++-
.../common/tools/TestZKIntegration.groovy | 54 +-
.../slider/providers/agent/AgentTestBase.groovy | 10 +
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 31 +-
.../test/YarnZKMiniClusterTestBase.groovy | 2 +-
.../agent/TestAgentProviderService.java | 29 +
.../services/workflow/EndOfServiceWaiter.java | 56 --
.../TestWorkflowForkedProcessService.java | 4 +-
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../flexing/TestClusterFlex1To1.groovy | 2 +-
75 files changed, 1509 insertions(+), 564 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index 61bf5dd,543c1a8..3145ecb
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@@ -25,14 -25,17 +25,19 @@@ import org.apache.slider.server.appmast
import java.util.concurrent.TimeUnit;
+ /**
+ * Notify the app master that it should register a component instance
+ * in the registry
+ * {@link SliderAppMaster#registerComponent(ContainerId)}
+ */
public class RegisterComponentInstance extends AsyncAction {
-
public final ContainerId containerId;
+ public final String description;
- public RegisterComponentInstance(ContainerId containerId, long delay,
+ public RegisterComponentInstance(ContainerId containerId,
+ String description,
+ long delay,
TimeUnit timeUnit) {
super("RegisterComponentInstance :" + containerId,
delay, timeUnit);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 3dc1d6d,97cc853..d80ee62
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@@ -246,9 -244,12 +243,11 @@@ public abstract class YarnMiniClusterTe
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
name = buildClustername(name)
- miniCluster =
- new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs, 1, false, false)
- miniCluster.init(conf)
+ miniCluster = new MiniYARNCluster(
+ name,
+ noOfNodeManagers,
+ numLocalDirs,
+ numLogDirs)
- miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
createMiniHDFSCluster(name, conf)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
[02/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e883bf9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e883bf9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e883bf9
Branch: refs/heads/develop
Commit: 6e883bf91de781fa8ec5ca7bb1f98f2b9fd8a0dd
Parents: c5fb4f0 82cf1f0
Author: Steve Loughran <st...@apache.org>
Authored: Mon Sep 1 16:55:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Sep 1 16:55:59 2014 +0100
----------------------------------------------------------------------
README.md | 4 +-
app-packages/hbase/appConfig.json | 74 +++++-----
app-packages/hbase/pom.xml | 2 +-
.../src/main/python/agent/Controller.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 10 +-
slider-agent/src/main/python/agent/Register.py | 5 +-
.../src/test/python/agent/TestRegistration.py | 5 +-
slider-assembly/src/main/bash/README.md | 2 +-
slider-assembly/src/main/bash/slider_destroy | 4 +-
slider-assembly/src/main/scripts/slider.py | 6 +-
.../org/apache/slider/client/SliderClient.java | 10 +-
.../slider/common/params/ActionFreezeArgs.java | 2 +-
.../common/params/LaunchArgsAccessor.java | 2 +-
.../slider/common/params/SliderActions.java | 8 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 147 ++++++++++++-------
.../server/appmaster/state/NodeEntry.java | 2 +-
.../server/appmaster/state/RoleHistory.java | 6 +-
.../appmaster/web/rest/agent/Register.java | 13 ++
.../agent/actions/TestActionExists.groovy | 2 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 10 +-
.../TestFreezeThawMasterlessAM.groovy | 8 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 2 +-
.../standalone/TestStandaloneAMDestroy.groovy | 6 +-
.../agent/standalone/TestYarnRegistryAM.groovy | 2 +-
.../slider/client/TestCommonArgParsing.groovy | 2 +-
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 13 +-
.../agent/TestAgentProviderService.java | 35 ++++-
.../src/test/resources/example-slider-test.xml | 4 +-
.../funtest/framework/CommandTestBase.groovy | 2 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 8 +-
.../src/test/manual/python/SliderTester.py | 4 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../funtest/HBaseClusterLifecycleIT.groovy | 8 +-
.../TestFreezeThawClusterFromArchive.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
.../TestFreezeThawLiveRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 4 +-
src/test/clusters/sandbox/operations.md | 20 +--
src/test/clusters/ubuntu-secure/operations.md | 14 +-
43 files changed, 286 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a91a817,61866fb..4bfc718
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -582,11 -589,11 +592,11 @@@ public class AgentProviderService exten
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
-
+
// and update registration entries
if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
- TimeUnit.MILLISECONDS));
+ queueAccess.put(new RegisterComponentInstance(instance.getId(),
+ roleName, 0, TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
index bdb5b27,0000000..bd15bb4
mode 100644,000000..100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
@@@ -1,379 -1,0 +1,379 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
+
+import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
+import org.apache.slider.core.registry.docstore.UriMap
+import org.apache.slider.core.registry.info.CustomRegistryConstants
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Test
+
+/**
+ * work with a YARN registry
+ */
+@CompileStatic
+@Slf4j
+
+class TestYarnRegistryAM extends AgentMiniClusterTestBase {
+
+
+ public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
+
+ @Test
+ public void testYarnRegistryAM() throws Throwable {
+
+
+ describe "create a masterless AM then perform YARN registry operations on it"
+
+
+ String clustername = createMiniCluster(configuration, 1, true)
+
+ // get local binding
+ def registryOperations = microZKCluster.registryOperations
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+
+ // verify the cluster has the YARN reg service live
+ def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
+ assert rmRegistryService
+
+
+
+
+
+ ServiceLauncher<SliderClient> launcher
+ launcher = createStandaloneAM(clustername, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)clusterNodes).size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def yarnRegistryClient = client.yarnAppListClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ // sleep to allow registration to complete
+ sleep(5000)
+
+
+
+
+ try {
+ def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ log.info("yarn service registry: \n${yarnRegistryDump}\n")
+ } catch (IOException ignored) {
+
+ }
+
+
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ def self = currentUser()
+ RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
+ dumpArray(serviceTypes)
+
+ def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
+
+ def serviceRecords = RecordOperations.extractServiceRecords(registryService,
+ registryService.listDir(recordsPath))
+ dumpCollection(serviceRecords)
+ assert serviceRecords.size() == 1
+
+ def serviceInstance = serviceRecords[0]
+ log.info(serviceInstance.toString())
+
+ assert 2 <= serviceInstance.external.size()
+
+ // hit the registry web page
+
+ def registryEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+ describe("Registry WADL @ $registryURL")
+
+ def publisherEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API)
+
+ def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
+ def setlisting = GET(publisherURL)
+
+ log.info(setlisting)
+
+ UriMap uris = uriMapDeser.fromJson(setlisting)
+ assert uris.uris[RestPaths.SLIDER_CONFIGSET]
+ def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
+
+ assert publishedYarnSite.empty
+
+ //get the full URL
+ def yarnSitePublisher = appendToURL(publisher,
+ RestPaths.SLIDER_CONFIGSET,
+ ARTIFACT_NAME)
+
+ String confJSON = GET(yarnSitePublisher)
+// log.info(confJSON)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+
+ publishedYarnSite = confSerDeser.fromJson(confJSON)
+
+ assert !publishedYarnSite.empty
+
+
+ //get the XML
+ def yarnSiteXML = yarnSitePublisher + ".xml"
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String properties = GET(yarnSitePublisher + ".properties")
+ Properties parsedProps = new Properties()
+ parsedProps.load(new StringReader(properties))
+ assert parsedProps.size() > 0
+ def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
+ def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
+ assert rmAddrFromDownloadedProperties
+ assert rmHostnameFromDownloadedProperties
+
+ String json = GET(yarnSitePublisher + ".json")
+
+
+
+ describe("Registry List")
+ log.info(GET(registryURL))
+
+
+ describe "Registry Retrieval Class"
+ // retrieval
+
+ RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[ARTIFACT_NAME]
+
+
+ describe "verify SLIDER-52 processing"
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ ARTIFACT_NAME,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
+ def rmHostnameViaClientSideXML = parsedProps.get(
+ YarnConfiguration.RM_HOSTNAME)
+ assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
+ def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
+ describe "fetch missing artifact"
+ try {
+ retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ try {
+ retriever.getConfigurations(false)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.verbose = true
+
+ // list all
+ registryArgs.list = true;
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+
+ // list a named instance and expect a failure
+ registryArgs.list = true;
+ registryArgs.name = "unknown"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ // list all instances of an alternate type and expect failure
+ registryArgs.list = true;
+ registryArgs.name = null
+ registryArgs.serviceType = "org-apache-hadoop"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ registryArgs.serviceType = ""
+
+ //set the name
+ registryArgs.name = clustername;
+ registryArgs.serviceType = SliderKeys.APP_TYPE
+
+
+ //now expect list to work
+ describe registryArgs.toString()
+
+ def listedInstance = client.actionRegistryListYarn(registryArgs)
+ assert listedInstance[0].id == serviceInstance.id
+
+
+ // listconf
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+ // listconf --internal
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ registryArgs.internal = true
+ describe registryArgs.toString()
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = false
+ registryArgs.internal = false
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registryArgs.getConf = yarn_site_config
+
+ //properties format
+ registryArgs.format = "properties"
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+
+ File outputDir = new File("target/test_standalone_registry_am/output")
+ outputDir.mkdirs()
+
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(clustername)
+ registryArgs.getConf = yarn_site_config
+ registryArgs.dest = outputDir
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".xml").exists()
+
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
+ describe registryArgs.toString()
+
+ def unknownFilename = "undefined-file"
+ registryArgs.getConf = unknownFilename
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
- describe "freeze cluster"
++ describe "stop cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = yarnRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e883bf9/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
----------------------------------------------------------------------
[40/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7f7bb5b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7f7bb5b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7f7bb5b8
Branch: refs/heads/develop
Commit: 7f7bb5b832a09d3420cc8a3c801b50a5f923e2b5
Parents: bfbc67a 0419a5a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 15:18:04 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 15:18:04 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 2 +-
app-packages/accumulo/pom.xml | 1 +
pom.xml | 6 +
slider-agent/conf/agent.ini | 1 +
.../src/main/python/agent/AgentConfig.py | 14 ++
.../src/main/python/agent/Controller.py | 35 +++-
.../src/test/python/agent/TestController.py | 63 ++++++
slider-agent/src/test/python/agent/TestMain.py | 37 ++++
.../org/apache/slider/common/SliderKeys.java | 4 +
.../slider/common/tools/CoreFileSystem.java | 15 ++
.../providers/agent/AgentProviderService.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 161 ++++++++++-----
.../security/SecurityConfiguration.java | 201 +++++++++++++++++++
.../security/SecurityConfigurationTest.groovy | 159 +++++++++++++++
.../AppsThroughAgentQueueAndLabelsIT.groovy | 26 +++
15 files changed, 702 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
[20/50] [abbrv] git commit: SLIDER-453 AM queue processing goes live
before init complete
Posted by st...@apache.org.
SLIDER-453 AM queue processing goes live before init complete
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/77ca3c27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/77ca3c27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/77ca3c27
Branch: refs/heads/develop
Commit: 77ca3c27acd5d0f3c683fc9be2556db827a3637b
Parents: b26863a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:59:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:59:01 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 29 ++++++++++++++++++--
.../slider/server/appmaster/state/AppState.java | 5 +++-
2 files changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/77ca3c27/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index fd7c360..2fa8342 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -291,6 +291,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
+ /**
+ * Flag set during the init process
+ */
+ private final AtomicBoolean initCompleted = new AtomicBoolean(false);
+
private volatile boolean success = true;
/**
@@ -434,11 +439,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+ }
+
+ /**
+ * Start the queue processing
+ */
+ private void startQueueProcessing() {
+ log.info("Queue Processing started");
executorService.execute(actionQueues);
executorService.execute(new QueueExecutor(this, actionQueues));
}
- /* =================================================================== */
+/* =================================================================== */
/* RunService methods called from ServiceLauncher */
/* =================================================================== */
@@ -822,7 +834,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
+ // declare the cluster initialized
+ log.info("Application Master Initialization Completed");
+ initCompleted.set(true);
+
+ // start handling any scheduled events
+
+ startQueueProcessing();
// Start the Slider AM provider
+
sliderAMProvider.start();
// launch the real provider; this is expected to trigger a callback that
@@ -1312,8 +1332,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.error("Role instance {} failed ", ri);
}
- getProviderService().notifyContainerCompleted(containerId);
- queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ // known nodes trigger notifications
+ if(!result.unknownNode) {
+ getProviderService().notifyContainerCompleted(containerId);
+ queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ }
}
reviewRequestAndReleaseNodes("onContainersCompleted");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/77ca3c27/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 57952e7..abb6fe8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1256,13 +1256,14 @@ public class AppState {
}
/**
- * This is a very small class to send a triple result back from
+ * This is a very small class to send a multiple result back from
* the completion operation
*/
public static class NodeCompletionResult {
public boolean surplusNode = false;
public RoleInstance roleInstance;
public boolean containerFailed;
+ public boolean unknownNode = false;
public String toString() {
@@ -1271,6 +1272,7 @@ public class AppState {
sb.append("surplusNode=").append(surplusNode);
sb.append(", roleInstance=").append(roleInstance);
sb.append(", containerFailed=").append(containerFailed);
+ sb.append(", unknownNode=").append(unknownNode);
sb.append('}');
return sb.toString();
}
@@ -1354,6 +1356,7 @@ public class AppState {
log.error("Notified of completed container {} that is not in the list" +
" of active or failed containers", containerId);
completionOfUnknownContainerEvent.incrementAndGet();
+ result.unknownNode = true;
}
}
[14/50] [abbrv] git commit: SLIDER-149 fix a couple of regressions
that showed up in testing ... this branch got out of sync with develop
Posted by st...@apache.org.
SLIDER-149 fix a couple of regressions that showed up in testing ... this branch got out of sync with develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dbd1d3d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dbd1d3d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dbd1d3d2
Branch: refs/heads/develop
Commit: dbd1d3d2dfda673c8db2792a457ab2cfb3930d65
Parents: 8a48c80
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 17 22:20:31 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 17 22:20:31 2014 +0100
----------------------------------------------------------------------
.../main/java/org/apache/slider/client/SliderClient.java | 6 ++----
.../org/apache/slider/core/registry/YarnAppListClient.java | 9 ++++++++-
2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dbd1d3d2/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8c07c5d..93050a5 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -217,10 +217,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
super.serviceInit(conf);
- //here the superclass is inited; getConfig returns a non-null value
- sliderFileSystem = new SliderFileSystem(getConfig());
- YarnAppListClient =
- new YarnAppListClient(yarnClient, getUsername(), getConfig());
}
/**
@@ -383,6 +379,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnClient.start();
}
addService(yarnClient);
+ YarnAppListClient =
+ new YarnAppListClient(yarnClient, getUsername(), getConfig());
// create the filesystem
sliderFileSystem = new SliderFileSystem(getConfig());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dbd1d3d2/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 0ca0518..068b687 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.registry;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -40,6 +41,13 @@ public class YarnAppListClient {
public YarnAppListClient(SliderYarnClientImpl yarnClient,
String username,
Configuration conf) {
+
+ Preconditions.checkArgument(yarnClient != null,
+ "yarn client is null: is app inited?");
+ Preconditions.checkArgument(username != null,
+ "username is null");
+ Preconditions.checkArgument(conf != null,
+ "conf parameter is null");
this.yarnClient = yarnClient;
this.username = username;
this.conf = conf;
@@ -54,7 +62,6 @@ public class YarnAppListClient {
*/
public List<ApplicationReport> findAllLiveInstances(String appname)
throws YarnException, IOException {
-
return yarnClient.findAllLiveInstances(username, appname);
}
[48/50] [abbrv] git commit: SLIDER-149 AM cleans entries under AM
registry entry on initial startup
Posted by st...@apache.org.
SLIDER-149 AM cleans entries under AM registry entry on initial startup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66957d4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66957d4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66957d4f
Branch: refs/heads/develop
Commit: 66957d4fcd428343f883959a6ecb927745b44fee
Parents: 7887fd3
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 15:36:50 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 15:36:50 2014 -0700
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 35 ++++++-----
.../YarnRegistryViewForProviders.java | 62 ++++++++++++++++----
.../providers/hbase/HBaseProviderService.java | 2 +-
3 files changed, 72 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8526207..a920a3e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -816,10 +816,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, actionQueues,
- liveContainers);
- sliderAMProvider.bind(stateForProviders, actionQueues,
- liveContainers);
+ providerService.bind(stateForProviders, actionQueues, liveContainers);
+ sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers);
// chaos monkey
maybeStartMonkey();
@@ -885,7 +883,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Ensure that the user is generated from a keytab and has no HDFS delegation
* tokens.
*
- * @param user
+ * @param user user to validate
* @throws SliderException
*/
protected void validateLoginUser(UserGroupInformation user)
@@ -939,16 +937,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
- appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT, agentWebApp.getPort());
+ appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
+ agentWebApp.getPort());
appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
- agentWebApp.getSecuredPort());
+ agentWebApp.getSecuredPort());
}
/**
* This registers the service instance and its external values
* @param instanceName name of this instance
* @param appid application ID
- * @throws Exception
+ * @throws IOException
*/
private void registerServiceInstance(String instanceName,
ApplicationId appid) throws IOException {
@@ -958,8 +957,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
URL amWebURI = new URL(appMasterTrackingUrl);
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
- String serviceName = SliderKeys.APP_TYPE;
- int id = appid.getId();
//Give the provider restricted access to the state, registry
setupInitialRegistryPaths();
@@ -998,10 +995,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// store for clients
log.info("Service Record \n{}", serviceRecord);
- yarnRegistryOperations.putService(service_user_name,
+ String sliderServicePath = yarnRegistryOperations.putService(service_user_name,
SliderKeys.APP_TYPE,
instanceName,
- serviceRecord);
+ serviceRecord, true);
+ boolean isFirstAttempt = 1 == appAttemptID.getAttemptId();
+ // delete the children in case there are any and this is an AM startup.
+ // just to make sure everything underneath is purged
+ if (isFirstAttempt) {
+ yarnRegistryOperations.deleteChildren(sliderServicePath, true);
+ }
yarnRegistryOperations.setSelfRegistration(serviceRecord);
// and a shorter lived binding to the app
@@ -1080,7 +1083,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
String cid = RegistryPathUtils.encodeYarnID(id.toString());
try {
- yarnRegistryOperations.rmComponent(cid);
+ yarnRegistryOperations.deleteComponent(cid);
} catch (IOException e) {
log.warn("Failed to delete container {} : {}", id, e, e);
}
@@ -1492,9 +1495,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onShutdownRequest() {
LOG_YARN.info("Shutdown Request received");
signalAMComplete(new ActionStopSlider("stop",
- EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- "Shutdown requested from RM"));
+ EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ "Shutdown requested from RM"));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index 4eb91ab..4104b16 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -18,6 +18,8 @@
package org.apache.slider.server.services.yarnregistry;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.registry.client.api.BindFlags;
import org.apache.hadoop.registry.client.api.RegistryOperations;
@@ -27,6 +29,11 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
public class YarnRegistryViewForProviders {
@@ -111,21 +118,28 @@ public class YarnRegistryViewForProviders {
}
/**
- * Add a service under a path
+ * Add a service under a path, optionally purging any history
* @param username user
* @param serviceClass service class to use under ~user
* @param serviceName name of the service
* @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first.
+ * @return the path the service was created at
* @throws IOException
*/
- public void putService(String username,
+ public String putService(String username,
String serviceClass,
String serviceName,
- ServiceRecord record) throws IOException {
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
String path = RegistryUtils.servicePath(
username, serviceClass, serviceName);
+ if (deleteTreeFirst) {
+ registryOperations.delete(path, true);
+ }
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ return path;
}
/**
@@ -133,22 +147,50 @@ public class YarnRegistryViewForProviders {
* @param serviceClass service class to use under ~user
* @param serviceName name of the service
* @param record service record
+ * @param deleteTreeFirst perform recursive delete of the path first
+ * @return the path the service was created at
* @throws IOException
*/
- public void putService(
+ public String putService(
String serviceClass,
String serviceName,
- ServiceRecord record) throws IOException {
- String path = RegistryUtils.servicePath(
- user, serviceClass, serviceName);
- registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.bind(path, record, BindFlags.OVERWRITE);
+ ServiceRecord record,
+ boolean deleteTreeFirst) throws IOException {
+ return putService(user, serviceClass, serviceName, record, deleteTreeFirst);
}
- public void rmComponent(String componentName) throws IOException {
+ /**
+ * Delete a component
+ * @param componentName component name
+ * @throws IOException
+ */
+ public void deleteComponent(String componentName) throws IOException {
String path = RegistryUtils.componentPath(
user, sliderServiceclass, instanceName,
componentName);
registryOperations.delete(path, false);
}
+
+ /**
+ * Delete the children of a path -but not the path itself.
+ * It is not an error if the path does not exist
+ * @param path path to delete
+ * @param recursive flag to request recursive deletes
+ * @throws IOException IO problems
+ */
+ public void deleteChildren(String path, boolean recursive) throws IOException {
+ List<String> childNames = null;
+ try {
+ childNames = registryOperations.list(path);
+ } catch (PathNotFoundException e) {
+ //ignored
+ }
+ Map<String, RegistryPathStatus> results =
+ new HashMap<String, RegistryPathStatus>();
+ for (String childName : childNames) {
+ String child = join(path, childName);
+ registryOperations.delete(child, recursive);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66957d4f/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index a36ae70..88e1f9e 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -275,7 +275,7 @@ public class HBaseProviderService extends AbstractProviderService
log.warn("failed to create config URL: {}", e, e);
}
log.info("registering {}/{}", name, HBASE_SERVICE_TYPE);
- yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord);
+ yarnRegistry.putService(HBASE_SERVICE_TYPE, name, serviceRecord, true);
PublishedConfiguration publishedSite =
new PublishedConfiguration("HBase site", siteConf);
[13/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8a48c802
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8a48c802
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8a48c802
Branch: refs/heads/develop
Commit: 8a48c802e2faee02255108f34af3f7b6eb39665d
Parents: 373f0c4 26e7c2a
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 17 22:02:49 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 17 22:02:49 2014 +0100
----------------------------------------------------------------------
.../main/python/jinja2/ext/Vim/htmljinja.vim | 27 -
.../src/main/python/jinja2/ext/Vim/jinja.vim | 113 ---
.../jinja2/ext/django2jinja/django2jinja.py | 768 -------------------
.../python/jinja2/ext/django2jinja/example.py | 7 -
.../ext/django2jinja/templates/index.html | 58 --
.../ext/django2jinja/templates/layout.html | 4 -
.../ext/django2jinja/templates/subtemplate.html | 1 -
.../src/main/python/jinja2/ext/djangojinja2.py | 86 ---
.../src/main/python/jinja2/ext/inlinegettext.py | 78 --
.../src/main/python/jinja2/ext/jinja.el | 213 -----
slider-assembly/pom.xml | 2 +-
.../appmaster/model/mock/MockContainerId.groovy | 5 +
12 files changed, 6 insertions(+), 1356 deletions(-)
----------------------------------------------------------------------
[24/50] [abbrv] git commit: SLIDER-149 migration off
curator-x-discovery: read only REST API (work in progress)
Posted by st...@apache.org.
SLIDER-149 migration off curator-x-discovery: read only REST API (work in progress)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/44e61659
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/44e61659
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/44e61659
Branch: refs/heads/develop
Commit: 44e61659f0c9cefb16b2cbc1f63447ee4c96c4fc
Parents: 8c7f6b2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 25 10:50:08 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 25 10:50:08 2014 +0100
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 8 +-
.../org/apache/slider/client/SliderClient.java | 271 ++++---------------
.../slider/client/SliderClusterOperations.java | 1 -
.../slider/core/launch/AbstractLauncher.java | 8 +-
.../core/launch/JavaCommandLineBuilder.java | 4 +-
.../registry/info/CustomRegistryConstants.java | 1 +
.../core/registry/info/ServiceInstanceData.java | 1 +
.../registry/retrieve/RegistryRetriever.java | 10 -
.../providers/AbstractProviderService.java | 46 ++--
.../slider/providers/ProviderService.java | 11 +-
.../providers/agent/AgentProviderService.java | 22 +-
.../slideram/SliderAMProviderService.java | 37 +--
.../server/appmaster/SliderAppMaster.java | 82 ++----
.../server/appmaster/web/SliderAMWebApp.java | 22 +-
.../slider/server/appmaster/web/WebAppApi.java | 4 +
.../server/appmaster/web/WebAppApiImpl.java | 17 +-
.../appmaster/web/rest/AMWebServices.java | 9 +
.../server/appmaster/web/rest/RestPaths.java | 4 +-
.../web/rest/registry/RegistryResource.java | 130 +++++++++
.../server/services/curator/CuratorHelper.java | 11 -
.../server/services/curator/CuratorService.java | 1 +
.../curator/CuratorServiceInstance.java | 1 +
.../curator/CuratorServiceInstances.java | 1 +
.../server/services/curator/CuratorUriSpec.java | 1 +
.../services/curator/RegistryBinderService.java | 1 +
.../registry/RegistryDiscoveryContext.java | 1 +
.../registry/RegistryRestResources.java | 1 +
.../registry/RegistryServiceConstants.java | 1 +
.../registry/RegistryViewForProviders.java | 48 ----
.../registry/SliderRegistryService.java | 8 +-
.../utility/AbstractSliderLaunchedService.java | 52 +---
.../YarnRegistryViewForProviders.java | 32 ++-
.../standalone/TestStandaloneAgentAM.groovy | 13 -
.../TestStandaloneYarnRegistryAM.groovy | 46 ++--
.../model/mock/MockProviderService.groovy | 4 -
.../view/TestClusterSpecificationBlock.groovy | 7 +-
.../web/view/TestContainerStatsBlock.groovy | 7 +-
.../appmaster/web/view/TestIndexBlock.groovy | 8 +-
.../apache/slider/test/SliderTestUtils.groovy | 13 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 2 +-
.../publisher/TestAgentProviderService.java | 5 +-
.../minicluster/live/TestHBaseMaster.groovy | 33 +--
.../minicluster/live/TestTwoLiveClusters.groovy | 35 ++-
44 files changed, 409 insertions(+), 614 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 475c4e3..613769e 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -25,6 +25,7 @@ import org.apache.hadoop.security.ProviderUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.security.alias.CredentialProvider
import org.apache.hadoop.security.alias.CredentialProviderFactory
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
import org.apache.slider.accumulo.CustomAuthenticator
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
@@ -170,10 +171,9 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
int tries = 5
while (true) {
try {
- CuratorServiceInstance<ServiceInstanceData> instance =
- sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE, clusterName)
- ServiceInstanceData serviceInstanceData = instance.payload
- RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
+ ServiceRecord instance =
+ sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName)
+ RegistryRetriever retriever = new RegistryRetriever(instance)
PublishedConfiguration configuration = retriever.retrieveConfiguration(
retriever.getConfigurations(true), "quicklinks", true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1afd161..e68a7cb 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -40,10 +40,10 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.registry.client.binding.BindingUtils;
-import org.apache.hadoop.yarn.registry.client.binding.RecordOperations;
-import org.apache.hadoop.yarn.registry.client.binding.ZKPathDumper;
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import static org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils.*;
+
+import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
@@ -120,8 +120,6 @@ import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.services.curator.CuratorServiceInstance;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -176,13 +174,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private SliderYarnClientImpl yarnClient;
private YarnAppListClient YarnAppListClient;
private AggregateConf launchedInstanceDefinition;
- private SliderRegistryService registry;
+// private SliderRegistryService registry;
/**
* The YARN registry service
*/
- private RegistryOperationsService registryOperations;
+ private RegistryOperations registryOperations;
/**
* Constructor
@@ -394,7 +392,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Delete the zookeeper node associated with the calling user and the cluster
+ * TODO: YARN registry operations
**/
+ @Deprecated
@VisibleForTesting
public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
String user = getUsername();
@@ -1454,7 +1454,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
public String getUsername() throws IOException {
- return UserGroupInformation.getCurrentUser().getShortUserName();
+ return RegistryOperationUtils.currentUser();
}
/**
@@ -2407,58 +2407,32 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException Network or other problems
*/
@VisibleForTesting
- @Deprecated
- public List<ServiceInstanceData> actionRegistryList(
- ActionRegistryArgs registryArgs)
- throws YarnException, IOException {
- SliderRegistryService registryService = getRegistry();
- String serviceType = registryArgs.serviceType;
- String name = registryArgs.name;
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- registryService.findInstances(serviceType, name);
- int size = instances.size();
- if (size == 0) {
- throw new FileNotFoundException("No entries for servicetype "
- + serviceType
- + " name " + name);
- }
- List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
- for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
- ServiceInstanceData payload = instance.payload;
- logInstance(payload, registryArgs.verbose);
- sids.add(payload);
- }
- return sids;
- }
-
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return the instances (for tests)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- @VisibleForTesting
public Collection<ServiceRecord> actionRegistryListYarn(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
- RegistryOperationsService operations = getRegistryOperations();
+ RegistryOperations operations = getRegistryOperations();
Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
String serviceclassPath =
- BindingUtils.serviceclassPath(BindingUtils.currentUser(),
+ serviceclassPath(
+ currentUser(),
serviceType);
- RegistryPathStatus[] listDir;
- listDir = operations.list(serviceclassPath);
- if (listDir.length == 0) {
- throw new PathNotFoundException("records under "
- + serviceclassPath);
+
+ try {
+ Map<String, ServiceRecord> recordMap =
+ listServiceRecords(operations, serviceclassPath);
+ RegistryPathStatus[] listDir;
+ if (recordMap.isEmpty()) {
+ throw new UnknownApplicationInstanceException(
+ "No applications registered under " + serviceclassPath);
+ }
+ serviceRecords = recordMap.values();
+ } catch (PathNotFoundException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(),
+ e);
}
- serviceRecords =
- RecordOperations.extractServiceRecords(operations, listDir).values();
} else {
ServiceRecord instance = lookupServiceRecord(registryArgs);
serviceRecords = new ArrayList<ServiceRecord>(1);
@@ -2517,33 +2491,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- /**
- * list configs available for an instance
- *
- * @param registryArgs registry Arguments
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- public void actionRegistryListConfigs(ActionRegistryArgs registryArgs)
- throws YarnException, IOException {
- ServiceInstanceData instance = lookupInstance(registryArgs);
-
- RegistryRetriever retriever = new RegistryRetriever(instance);
- PublishedConfigSet configurations =
- retriever.getConfigurations(!registryArgs.internal);
-
- for (String configName : configurations.keys()) {
- if (!registryArgs.verbose) {
- log.info("{}", configName);
- } else {
- PublishedConfiguration published =
- configurations.get(configName);
- log.info("{} : {}",
- configName,
- published.description);
- }
- }
- }
/**
* list configs available for an instance
*
@@ -2582,31 +2529,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws FileNotFoundException if the config is not found
*/
@VisibleForTesting
- @Deprecated
- public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
- throws YarnException, IOException {
- ServiceInstanceData instance = lookupInstance(registryArgs);
-
- RegistryRetriever retriever = new RegistryRetriever(instance);
- boolean external = !registryArgs.internal;
- PublishedConfigSet configurations =
- retriever.getConfigurations(external);
-
- PublishedConfiguration published = retriever.retrieveConfiguration(configurations,
- registryArgs.getConf,
- external);
- return published;
- }
-
- /**
- * list configs available for an instance
- *
- * @param registryArgs registry Arguments
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- * @throws FileNotFoundException if the config is not found
- */
- @VisibleForTesting
public PublishedConfiguration actionRegistryGetConfigYarn(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
ServiceRecord instance = lookupServiceRecord(registryArgs);
@@ -2663,96 +2585,49 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Look up an instance
* @return instance data
- * @throws UnknownApplicationInstanceException no match
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
- private ServiceInstanceData lookupInstance(ActionRegistryArgs registryArgs) throws
- UnknownApplicationInstanceException,
+ private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
SliderException,
IOException {
- return lookupInstance(registryArgs.name, registryArgs.serviceType);
+ return lookupServiceRecord(registryArgs.serviceType, registryArgs.name);
}
/**
* Look up an instance
- * @param id instance ID
* @param serviceType service type
+ * @param id instance ID
* @return instance data
* @throws UnknownApplicationInstanceException no match
* @throws SliderException other failures
* @throws IOException IO problems or wrapped exceptions
*/
- private ServiceInstanceData lookupInstance(String id,
- String serviceType) throws
- IOException {
+ public ServiceRecord lookupServiceRecord(String serviceType, String id)
+ throws IOException, SliderException {
try {
- CuratorServiceInstance<ServiceInstanceData> csi =
- getRegistry().queryForInstance(serviceType, id);
- if (csi == null) {
- throw new FileNotFoundException(
- String.format("instance %s of type %s not found",
- id, serviceType));
- }
- return csi.getPayload();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e);
+ return getRegistryOperations().resolve(
+ servicePath(currentUser(),
+ serviceType, id));
+ } catch (PathNotFoundException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(), e);
}
- }
-
-
- /**
- * Look up an instance
- * @return instance data
- * @throws SliderException other failures
- * @throws IOException IO problems or wrapped exceptions
- */
- private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
- SliderException,
- IOException {
- return lookupServiceRecord(registryArgs.name, registryArgs.serviceType);
- }
-
- /**
- * Look up an instance
- * @param id instance ID
- * @param serviceType service type
- * @return instance data
- * @throws UnknownApplicationInstanceException no match
- * @throws SliderException other failures
- * @throws IOException IO problems or wrapped exceptions
- */
- private ServiceRecord lookupServiceRecord(String id,
- String serviceType) throws
- IOException, SliderException {
- return getRegistryOperations().resolve(
- BindingUtils.servicePath(BindingUtils.currentUser(),
- serviceType, id));
}
-
-
/**
- * List instances in the registry
- * @return
- * @throws IOException
+ * List instances in the registry for the current user
+ * @return a list of slider registry instances
+ * @throws IOException Any IO problem ... including no path in the registry
+ * to slider service classes for this user
* @throws YarnException
*/
- public List<CuratorServiceInstance<ServiceInstanceData>> listRegistryInstances()
- throws IOException, YarnException {
- return getRegistry().listInstances(SliderKeys.APP_TYPE);
- }
-
- /**
- * Get an on-demand path jumper
- * @return a class that can dump the contents of the registry
- */
- @VisibleForTesting
- public ZKPathDumper dumpSliderRegistry(boolean verbose) throws SliderException, IOException {
- return getRegistry().dumpPath(verbose);
+ public Map<String, ServiceRecord> listRegistryInstances()
+ throws IOException, YarnException {
+ Map<String, ServiceRecord> recordMap = listServiceRecords(
+ getRegistryOperations(),
+ serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
+ return recordMap;
}
/**
@@ -2765,7 +2640,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
IOException,
YarnException {
try {
- return getRegistry().instanceIDs(SliderKeys.APP_TYPE);
+ Map<String, ServiceRecord> recordMap = listServiceRecords(
+ getRegistryOperations(),
+ serviceclassPath(currentUser(), SliderKeys.APP_TYPE));
+ return new ArrayList<String>(recordMap.keySet());
/// JDK7 } catch (YarnException | IOException e) {
} catch (IOException e) {
throw e;
@@ -2782,39 +2660,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws SliderException
* @throws IOException
*/
- private synchronized SliderRegistryService maybeStartRegistry() throws
- SliderException,
- IOException {
-
- if (registry == null) {
- registry = startRegistrationService();
- }
- return registry;
- }
-
- /**
- * Get the registry binding. As this may start the registry, it can take time
- * and fail
- * @return registry the registry service
- * @throws SliderException slider-specific failures
- * @throws IOException other failures
- */
- @VisibleForTesting
-
- public SliderRegistryService getRegistry() throws
- SliderException,
- IOException {
- return maybeStartRegistry();
- }
-
-
- /**
- * Start the registry if it is not there yet
- * @return the registry service
- * @throws SliderException
- * @throws IOException
- */
- private synchronized RegistryOperationsService maybeStartYarnRegistry()
+ private synchronized RegistryOperations maybeStartYarnRegistry()
throws SliderException, IOException {
if (registryOperations == null) {
@@ -2824,26 +2670,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * Get the YARN registry
+ * Get the registry binding. As this may start the registry, it can take time
+ * and fail
* @return the registry
*/
- public RegistryOperationsService getRegistryOperations()
+ public RegistryOperations getRegistryOperations()
throws SliderException, IOException {
return maybeStartYarnRegistry();
}
-
- /**
- * Get an on-demand path jumper
- * @return a class that can dump the contents of the registry
- */
- @VisibleForTesting
- public ZKPathDumper dumpYarnRegistry(boolean verbose)
- throws SliderException, IOException {
- return getRegistryOperations().dumpPath();
- }
-
-
/**
* Output to standard out/stderr (implementation specific detail)
* @param src source
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
index 9e1f568..30f17b6 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClusterOperations.java
@@ -56,7 +56,6 @@ public class SliderClusterOperations {
/**
* Get a node from the AM
- * @param appMaster AM
* @param uuid uuid of node
* @return deserialized node
* @throws IOException IO problems
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 366fc8b..f343933 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -52,6 +52,7 @@ import java.util.Map;
public abstract class AbstractLauncher extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
+ public static final String CLASSPATH = "CLASSPATH";
/**
* Filesystem to use for the launch
*/
@@ -194,8 +195,6 @@ public abstract class AbstractLauncher extends Configured {
dumpLocalResources();
containerLaunchContext.setLocalResources(localResources);
-
-
//tokens
log.debug("{} tokens", credentials.numberOfTokens());
DataOutputBuffer dob = new DataOutputBuffer();
@@ -203,7 +202,6 @@ public abstract class AbstractLauncher extends Configured {
ByteBuffer tokenBuffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
containerLaunchContext.setTokens(tokenBuffer);
-
return containerLaunchContext;
}
@@ -248,7 +246,6 @@ public abstract class AbstractLauncher extends Configured {
public void extractResourceRequirements(Resource resource,
Map<String, String> map) {
-
if (map != null) {
MapOperations options = new MapOperations("", map);
resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
@@ -258,13 +255,12 @@ public abstract class AbstractLauncher extends Configured {
}
}
-
/**
* Utility method to set up the classpath
* @param classpath classpath to use
*/
public void setClasspath(ClasspathConstructor classpath) {
- setEnv("CLASSPATH", classpath.buildClasspath());
+ setEnv(CLASSPATH, classpath.buildClasspath());
}
public void setEnv(String var, String value) {
Preconditions.checkArgument(var != null, "null variable name");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 0367e06..0b3fa10 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -32,7 +32,6 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
add(getJavaBinary());
}
-
/**
* Get the java binary. This is called in the constructor so don't try and
* do anything other than return a constant.
@@ -53,6 +52,9 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
}
}
+ /**
+ * Turn Java assertions on
+ */
public void enableJavaAssertions() {
add("-ea");
add("-esa");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
index d2658c1..65c122f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -25,6 +25,7 @@ public class CustomRegistryConstants {
public static final String MANAGEMENT_REST_API =
"org.apache.slider.management";
+
public static final String REGISTRY_REST_API =
"org.apache.slider.registry";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
index c3c7e63..52ae2e2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
@@ -35,6 +35,7 @@ import java.util.Map;
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@Deprecated
public class ServiceInstanceData implements Serializable {
public String serviceType;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
index a05d5be..1d11ef3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/retrieve/RegistryRetriever.java
@@ -34,7 +34,6 @@ import org.apache.slider.core.exceptions.ExceptionConverter;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,13 +62,6 @@ public class RegistryRetriever {
jerseyClient = Client.create(clientConfig);
jerseyClient.setFollowRedirects(true);
}
-
-
- public RegistryRetriever(ServiceInstanceData instance) {
- externalConfigurationURL = instance.externalView.configurationsURL;
- internalConfigurationURL = instance.internalView.configurationsURL;
- }
-
public RegistryRetriever(String externalConfigurationURL, String internalConfigurationURL) {
this.externalConfigurationURL = externalConfigurationURL;
@@ -95,10 +87,8 @@ public class RegistryRetriever {
} else {
externalConfigurationURL = "";
}
-
}
-
/**
* Does a bonded registry retriever have a configuration?
* @param external flag to indicate that it is the external entries to fetch
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 0e7ea25..c223901 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -23,6 +23,10 @@ import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
+import org.apache.hadoop.yarn.registry.client.types.Endpoint;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
@@ -33,14 +37,11 @@ import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
@@ -50,6 +51,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -72,8 +74,6 @@ public abstract class AbstractProviderService
LoggerFactory.getLogger(AbstractProviderService.class);
protected StateAccessForProviders amState;
protected AgentRestOperations restOps;
- protected RegistryViewForProviders registry;
- protected ServiceInstanceData registryInstanceData;
protected URL amWebAPI;
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
@@ -102,11 +102,9 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers) {
this.amState = stateAccessor;
- this.registry = reg;
this.queueAccess = queueAccess;
}
@@ -325,27 +323,35 @@ public abstract class AbstractProviderService
@Override
public void buildEndpointDetails(Map<String, String> details) {
- ServiceInstanceData self = registry.getSelfRegistration();
- Map<String, RegisteredEndpoint> endpoints =
- self.getRegistryView(true).endpoints;
- for (Map.Entry<String, RegisteredEndpoint> endpoint : endpoints.entrySet()) {
- RegisteredEndpoint val = endpoint.getValue();
- if (val.type.equals(RegisteredEndpoint.TYPE_URL)) {
- details.put(val.description, val.address);
+ ServiceRecord self = yarnRegistry.getSelfRegistration();
+
+ List<Endpoint> externals = self.external;
+ for (Endpoint endpoint : externals) {
+ String addressType = endpoint.addressType;
+ if (AddressTypes.ADDRESS_URI.equals(addressType)) {
+ try {
+ List<URL> urls = RegistryTypeUtils.retrieveAddressURLs(endpoint);
+ if (!urls.isEmpty()) {
+ details.put(endpoint.api, urls.get(0).toString());
+ }
+ } catch (InvalidRecordException ignored) {
+ // Ignored
+ } catch (MalformedURLException ignored) {
+ // ignored
+ }
+
}
+
}
}
@Override
public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData registryInstanceData,
- ServiceRecord serviceRecord)
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
throws IOException {
-
this.amWebAPI = amWebURI;
- this.registryInstanceData = registryInstanceData;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index b28ac6a..d78ca0f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -32,13 +32,11 @@ import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.File;
@@ -160,7 +158,6 @@ public interface ProviderService extends ProviderCore,
Map<String, String> buildMonitorDetails(ClusterDescription clusterSpec);
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg,
QueueAccess queueAccess,
List<Container> liveContainers);
@@ -187,14 +184,12 @@ public interface ProviderService extends ProviderCore,
* @param amWebURI
* @param agentOpsURI
* @param agentStatusURI
- * @param registryInstanceData
* @param serviceRecord
*/
void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData registryInstanceData,
- ServiceRecord serviceRecord)
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 00a1b78..5de08dc 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -51,8 +51,6 @@ import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
@@ -629,31 +627,19 @@ public class AgentProviderService extends AbstractProviderService implements
}
@Override
- public void applyInitialRegistryDefinitions (URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceInstanceData instanceData,
- ServiceRecord serviceRecord)
+ public void applyInitialRegistryDefinitions(URL amWebURI,
+ URL agentOpsURI,
+ URL agentStatusURI,
+ ServiceRecord serviceRecord)
throws IOException {
super.applyInitialRegistryDefinitions(amWebURI,
agentOpsURI,
agentStatusURI,
- instanceData,
serviceRecord);
try {
URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
-
- instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_SECURE_REST_API,
- new RegisteredEndpoint(restURL,
- "Agent Secure REST API"));
URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
- instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_ONEWAY_REST_API,
- new RegisteredEndpoint(
- agentStatusURL,
- "Agent Oneway REST API"));
serviceRecord.addInternalEndpoint(
new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index f01f995..72322c7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -36,9 +36,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.RegistryView;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
@@ -56,8 +53,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_MANAGEMENT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* Exists just to move some functionality out of AppMaster into a peer class
@@ -115,13 +111,11 @@ public class SliderAMProviderService extends AbstractProviderService implements
public void applyInitialRegistryDefinitions(URL amWebURI,
URL agentOpsURI,
URL agentStatusURI,
- ServiceInstanceData instanceData,
ServiceRecord serviceRecord)
throws IOException {
super.applyInitialRegistryDefinitions(amWebURI,
agentOpsURI,
agentStatusURI,
- instanceData,
serviceRecord);
// now publish site.xml files
YarnConfiguration defaultYarnConfig = new YarnConfiguration();
@@ -129,8 +123,7 @@ public class SliderAMProviderService extends AbstractProviderService implements
PublishedArtifacts.COMPLETE_CONFIG,
new PublishedConfiguration(
"Complete slider application settings",
- getConfig(), getConfig())
- );
+ getConfig(), getConfig()));
amState.getPublishedSliderConfigurations().put(
PublishedArtifacts.YARN_SITE_CONFIG,
new PublishedConfiguration(
@@ -153,40 +146,16 @@ public class SliderAMProviderService extends AbstractProviderService implements
try {
- RegisteredEndpoint webUI =
- new RegisteredEndpoint(amWebURI, "Application Master Web UI");
URL managementAPI = new URL(amWebURI, SLIDER_PATH_MANAGEMENT);
- URL registryREST = new URL(amWebURI, RestPaths.SLIDER_PATH_REGISTRY + "/" +
- RestPaths.REGISTRY_SERVICE);
-
- RegistryView externalView = instanceData.externalView;
- externalView.endpoints.put(CustomRegistryConstants.WEB_UI, webUI);
-
- externalView.endpoints.put(
- CustomRegistryConstants.MANAGEMENT_REST_API,
- new RegisteredEndpoint(
- managementAPI,
- "Management REST API") );
-
- externalView.endpoints.put(
- CustomRegistryConstants.REGISTRY_REST_API,
- new RegisteredEndpoint(
- registryREST,
- "Registry Web Service" ) );
+ URL registryREST = new URL(amWebURI, SLIDER_PATH_REGISTRY );
URL publisherURL = new URL(amWebURI, SLIDER_PATH_PUBLISHER);
- externalView.endpoints.put(
- CustomRegistryConstants.PUBLISHER_REST_API,
- new RegisteredEndpoint(
- publisherURL,
- "Publisher Service") );
// Set the configurations URL.
String configurationsURL = SliderUtils.appendToURL(
publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET);
- externalView.configurationsURL = configurationsURL;
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.webEndpoint(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 754013a..df54cfa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
@@ -95,9 +95,7 @@ import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.RegistryNaming;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
@@ -137,7 +135,6 @@ import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.security.FsDelegationTokenManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
@@ -182,6 +179,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceStateChangeListener,
RoleKeys,
ProviderCompleted {
+
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -204,8 +202,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
- public static final String SLIDER_AM_RPC = "Slider AM RPC";
- public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/**
* Singleton of metrics registry
@@ -327,16 +323,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private ProviderService providerService;
/**
- * The registry service
- */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private SliderRegistryService registry;
-
- /**
* The YARN registry service
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private RegistryOperationsService registryOperations;
+ private RegistryOperations registryOperations;
/**
* Record of the max no. of cores allowed in this cluster
@@ -648,12 +638,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
-
- //registry
- log.info("Starting slider registry");
- registry = startRegistrationService();
- log.info(registry.toString());
-
log.info("Starting Yarn registry");
registryOperations = startRegistryOperationsService();
log.info(registryOperations.toString());
@@ -671,12 +655,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startAgentWebApp(appInformation, serviceConf);
- webApp = new SliderAMWebApp(registry, registryOperations);
+ webApp = new SliderAMWebApp(registryOperations);
WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class,
new WebAppApiImpl(this,
stateForProviders,
providerService,
- certificateManager),
+ certificateManager, registryOperations),
RestPaths.WS_CONTEXT)
.with(serviceConf)
.start(webApp);
@@ -790,9 +774,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, actionQueues,
+ providerService.bind(stateForProviders, actionQueues,
liveContainers);
- sliderAMProvider.bind(stateForProviders, registry, actionQueues,
+ sliderAMProvider.bind(stateForProviders, actionQueues,
liveContainers);
// chaos monkey
@@ -873,7 +857,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
new WebAppApiImpl(this,
stateForProviders,
providerService,
- certificateManager),
+ certificateManager, registryOperations),
RestPaths.AGENT_WS_CONTEXT)
.withComponentConfig(getInstanceDefinition().getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM))
@@ -930,22 +914,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
providerService.bindToYarnRegistry(yarnRegistryOperations);
sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations);
- List<String> serviceInstancesRunning = registry.instanceIDs(serviceName);
- log.info("service instances already running: {}", serviceInstancesRunning);
-
-
- // slider instance data
- ServiceInstanceData instanceData = new ServiceInstanceData(registryId,
- serviceType);
-
-
- // IPC services
- instanceData.externalView.endpoints.put(
- CustomRegistryConstants.AM_IPC_PROTOCOL,
- new RegisteredEndpoint(rpcServiceAddress,
- RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF,
- SLIDER_AM_RPC) );
-
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
String serviceID = appid.toString();
@@ -960,38 +928,34 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
RegistryTypeUtils.marshall(rpcServiceAddress)));
// internal services
-
-
sliderAMProvider.applyInitialRegistryDefinitions(amWebURI,
- agentOpsURI,
- agentStatusURI,
- instanceData,
- serviceRecord);
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
// provider service dynamic definitions.
providerService.applyInitialRegistryDefinitions(amWebURI,
- agentOpsURI,
- agentStatusURI,
- instanceData,
- serviceRecord);
-
+ agentOpsURI,
+ agentStatusURI,
+ serviceRecord);
- // push the registration info to ZK
- registry.registerSelf(
- instanceData, amWebURI
- );
+ // store for clients
log.info("Service Record \n{}", serviceRecord);
yarnRegistryOperations.putService(service_user_name,
SliderKeys.APP_TYPE,
instanceName,
serviceRecord);
+ yarnRegistryOperations.setSelfRegistration(serviceRecord);
- // and an ephemeral binding to the app
+ // and a shorter lived binding to the app
+ String attempt = appAttemptID.toString();
+ ServiceRecord attemptRecord = new ServiceRecord(serviceRecord);
+ attemptRecord.id = attempt;
+ attemptRecord.persistence = PersistencePolicies.APPLICATION_ATTEMPT;
yarnRegistryOperations.putComponent(
- RegistryPathUtils.encodeYarnID(appAttemptID.toString()),
- serviceRecord
- );
+ RegistryPathUtils.encodeYarnID(attempt),
+ serviceRecord);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 36e1dad..64be603 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -22,6 +22,7 @@ import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
import org.apache.hadoop.yarn.webapp.Dispatcher;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
@@ -30,6 +31,7 @@ import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
+import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
import org.apache.slider.server.services.curator.CuratorHelper;
import org.apache.slider.server.services.registry.RegistryDiscoveryContext;
import org.apache.slider.server.services.registry.RegistryRestResources;
@@ -48,13 +50,9 @@ public class SliderAMWebApp extends WebApp {
public static final String CONTAINER_STATS = "/stats";
public static final String CLUSTER_SPEC = "/spec";
- public final SliderRegistryService registry;
- private final RegistryOperationsService yarnRegistry;
+ private final RegistryOperations registry;
-
- public SliderAMWebApp(SliderRegistryService registry,
- RegistryOperationsService yarnRegistry) {
- this.yarnRegistry = yarnRegistry;
+ public SliderAMWebApp(RegistryOperations registry) {
Preconditions.checkArgument(registry != null, "registry null");
this.registry = registry;
}
@@ -71,18 +69,6 @@ public class SliderAMWebApp extends WebApp {
// bind the REST interface
bind(AMWebServices.class);
//bind(AMAgentWebServices.class);
-
- CuratorHelper curatorHelper = registry.getCuratorHelper();
- ServiceDiscovery<ServiceInstanceData> discovery = registry.getDiscovery();
- RegistryDiscoveryContext discoveryContext = curatorHelper
- .createDiscoveryContext(
- discovery);
-
- bind(RegistryDiscoveryContext.class).toInstance(discoveryContext);
- RegistryRestResources registryRestResources =
- new RegistryRestResources(discoveryContext, registry);
- bind(RegistryRestResources.class).toInstance(registryRestResources);
-
route("/", SliderAMController.class);
route(CONTAINER_STATS, SliderAMController.class, "containerStats");
route(CLUSTER_SPEC, SliderAMController.class, "specification");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index aa20baa..8d91fd4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.server.appmaster.web;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.AppState;
@@ -62,4 +63,7 @@ public interface WebAppApi {
* Returns an interface that can support the agent-based REST operations.
*/
public AgentRestOperations getAgentRestOperations();
+
+
+ RegistryOperations getRegistryOperations();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 4d595a9..52b53a9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.server.appmaster.web;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.ProviderRole;
@@ -47,11 +48,14 @@ public class WebAppApiImpl implements WebAppApi {
protected final StateAccessForProviders appState;
protected final ProviderService provider;
protected final CertificateManager certificateManager;
-
+ private final RegistryOperations registryOperations;
+
public WebAppApiImpl(SliderClusterProtocol clusterProto,
- StateAccessForProviders appState,
- ProviderService provider,
- CertificateManager certificateManager) {
+ StateAccessForProviders appState,
+ ProviderService provider,
+ CertificateManager certificateManager,
+ RegistryOperations registryOperations) {
+ this.registryOperations = registryOperations;
checkNotNull(clusterProto);
checkNotNull(appState);
checkNotNull(provider);
@@ -145,4 +149,9 @@ public class WebAppApiImpl implements WebAppApi {
public AgentRestOperations getAgentRestOperations() {
return provider.getAgentRestOperations();
}
+
+ @Override
+ public RegistryOperations getRegistryOperations() {
+ return registryOperations;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 4f068f3..30db98e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -22,6 +22,7 @@ import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
+import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource;
import javax.ws.rs.Path;
@@ -29,6 +30,7 @@ import javax.ws.rs.Path;
@Singleton
@Path(RestPaths.SLIDER_CONTEXT_ROOT)
public class AMWebServices {
+
/** AM/WebApp info object */
private WebAppApi slider;
@@ -46,4 +48,11 @@ public class AMWebServices {
public PublisherResource getPublisherResource() {
return new PublisherResource(slider);
}
+
+ @Path(RestPaths.SLIDER_SUBPATH_REGISTRY)
+ public RegistryResource getRegistryResource() {
+ return new RegistryResource(slider);
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 0571ca1..93601ad 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -42,10 +42,12 @@ public class RestPaths {
+ SLIDER_SUBPATH_PUBLISHER;
public static final String SLIDER_SUBPATH_REGISTRY = "/registry";
- public static final String SLIDER_PATH_REGISTRY = WS_CONTEXT_ROOT
+ public static final String SLIDER_PATH_REGISTRY = SLIDER_CONTEXT_ROOT
+ SLIDER_SUBPATH_REGISTRY;
+ @Deprecated
public static final String REGISTRY_SERVICE = "v1/service";
+ @Deprecated
public static final String REGISTRY_ANYSERVICE = "v1/anyservice";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
new file mode 100644
index 0000000..67efa0d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/registry/RegistryResource.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.registry;
+
+import com.google.inject.Singleton;
+import org.apache.hadoop.fs.PathAccessDeniedException;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.exceptions.AuthenticationFailedException;
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
+import java.util.List;
+
+/**
+ * This is the read-only view of the slider YARN registry
+ *
+ * Model:
+ * <ol>
+ * <li>a tree of nodes</li>
+ * <li>Default view is of children + record</li>
+ * </ol>
+ *
+ */
+@Singleton
+//@Path(RestPaths.SLIDER_PATH_REGISTRY)
+public class RegistryResource {
+ protected static final Logger log =
+ LoggerFactory.getLogger(RegistryResource.class);
+ public static final String SERVICE_PATH =
+ "/{path:.*}";
+
+ private final RegistryOperations registry;
+
+ /**
+ * Construct an instance bonded to a registry
+ * @param slider slider API
+ */
+ public RegistryResource(WebAppApi slider) {
+ this.registry = slider.getRegistryOperations();
+ }
+
+ /**
+ * Internal init code, per request
+ * @param request incoming request
+ * @param uriInfo URI details
+ */
+ private void init(HttpServletRequest request, UriInfo uriInfo) {
+ log.debug(uriInfo.getRequestUri().toString());
+ }
+
+ @GET
+ public Response getRoot(@Context HttpServletRequest request) {
+ return Response.ok("registry root").build();
+ }
+
+
+// {path:.*}
+
+ @Path(SERVICE_PATH)
+ @GET
+ @Produces({MediaType.APPLICATION_JSON})
+ public Response lookup(
+ @PathParam("path") String path,
+ @Context HttpServletRequest request,
+ @Context UriInfo uriInfo) {
+ try {
+ init(request, uriInfo);
+ List<RegistryPathStatus> list = registry.listFull(path);
+ return Response.ok("found").build();
+ } catch (PathNotFoundException e) {
+ throw new NotFoundException(path);
+ } catch (AuthenticationFailedException e) {
+ throw new ForbiddenException(path);
+ } catch (PathAccessDeniedException e) {
+ throw new ForbiddenException(path);
+ } catch (Exception e) {
+ return fromException(e);
+ }
+ }
+
+ /**
+ * Handle an exception
+ * @param e exception
+ * @return a response to return
+ */
+ Response fromException(Exception e) {
+ log.error("Error during generation of response: {}", e, e);
+ if (e instanceof PathNotFoundException) {
+ return Response.status(Response.Status.NOT_FOUND).build();
+ }
+ if (e instanceof AuthenticationFailedException
+ || e instanceof PathAccessDeniedException) {
+ return Response.status(Response.Status.FORBIDDEN).build();
+ }
+ return Response.serverError().build();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
index 72d413f..9e6f53a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
@@ -75,17 +75,6 @@ public class CuratorHelper extends Configured {
}
/**
- * Create an (united) curator client service
- * @param connectionString ZK binding
- * @return the service
- */
- public CuratorService createCuratorClientService() {
- CuratorService curatorService =
- new CuratorService("Curator ", curator, connectionString);
- return curatorService;
- }
-
- /**
* Create a discovery builder bonded to this curator
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
index c3ace4d..a98a192 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
+@Deprecated
public class CuratorService extends AbstractService {
private static final Logger log =
LoggerFactory.getLogger(CuratorService.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
index 61efde2..0acadb2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstance.java
@@ -24,6 +24,7 @@ import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonTypeInfo;
@JsonIgnoreProperties(ignoreUnknown = true)
+@Deprecated
public class CuratorServiceInstance<T> {
public String name;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
index 8923e63..2b575be 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorServiceInstances.java
@@ -24,6 +24,7 @@ import java.util.List;
/**
*
*/
+@Deprecated
public class CuratorServiceInstances<T> {
private final List<CuratorServiceInstance<T>> services;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
index adda359..478d3f4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorUriSpec.java
@@ -26,6 +26,7 @@ import java.util.List;
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
+@Deprecated
public class CuratorUriSpec extends UriSpec{
private final List<Part> parts = Lists.newArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
index dbdcee4..fca967b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -46,6 +46,7 @@ import java.util.Map;
* start/close methods are tied to the lifecycle of this service
* @param <Payload> the payload of the operation
*/
+@Deprecated
public class RegistryBinderService<Payload> extends CuratorService {
private static final Logger log =
LoggerFactory.getLogger(RegistryBinderService.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
index a7c35e8..d6b676f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryDiscoveryContext.java
@@ -23,6 +23,7 @@ import org.apache.curator.x.discovery.ServiceDiscovery;
import org.apache.curator.x.discovery.server.contexts.GenericDiscoveryContext;
import org.apache.slider.core.registry.info.ServiceInstanceData;
+@Deprecated
public class RegistryDiscoveryContext extends GenericDiscoveryContext<ServiceInstanceData> {
public RegistryDiscoveryContext(ServiceDiscovery<ServiceInstanceData> serviceDiscovery,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
index e4e8523..ea97847 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
@@ -48,6 +48,7 @@ import java.util.Random;
@Singleton
@Path(RestPaths.SLIDER_PATH_REGISTRY)
+@Deprecated
public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData> {
public static final String SERVICE_NAME = RestPaths.REGISTRY_SERVICE +"/{name}";
public static final String SERVICE_NAME_ID = SERVICE_NAME + "/{id}";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
index ee24dc1..f3a107f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryServiceConstants.java
@@ -21,6 +21,7 @@ package org.apache.slider.server.services.registry;
/**
* These constants are unique to the slider registry service itself
*/
+@Deprecated
public class RegistryServiceConstants {
public static final int INSTANCE_REFRESH_MS = 1000;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java
deleted file mode 100644
index 22ba066..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryViewForProviders.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.registry;
-
-import org.apache.slider.core.registry.info.ServiceInstanceData;
-
-import java.io.IOException;
-import java.net.URL;
-import java.util.List;
-
-/**
- * This offers restricted access to the registry for providers
- */
-public interface RegistryViewForProviders {
- List<ServiceInstanceData> listInstancesByType(String serviceType) throws
- IOException;
-
- /**
- * Get the registration of slider itself
- * @return the registration of slider
- */
- ServiceInstanceData getSelfRegistration();
-
- /**
- * Register the service, raising IOExceptions when anything fails
- * @param instanceData instance data
- * @param url URL to register
- * @throws IOException on registration problems
- */
- void registerServiceInstance(
- ServiceInstanceData instanceData, URL url) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
index ca4d180..ab92c82 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
@@ -34,10 +34,9 @@ import java.util.List;
* This is the registry service, which tries to hide exactly how the
* registry is implemented
*/
-
+@Deprecated
public class SliderRegistryService
- extends RegistryBinderService<ServiceInstanceData>
- implements RegistryViewForProviders {
+ extends RegistryBinderService<ServiceInstanceData> {
private ServiceInstanceData selfRegistration;
@@ -48,7 +47,6 @@ public class SliderRegistryService
}
- @Override
public List<ServiceInstanceData> listInstancesByType(String serviceType) throws
IOException {
List<CuratorServiceInstance<ServiceInstanceData>> services =
@@ -60,7 +58,6 @@ public class SliderRegistryService
return payloads;
}
- @Override
public ServiceInstanceData getSelfRegistration() {
return selfRegistration;
}
@@ -81,7 +78,6 @@ public class SliderRegistryService
setSelfRegistration(instanceData);
}
- @Override
public void registerServiceInstance(
ServiceInstanceData instanceData, URL url) throws IOException {
Preconditions.checkNotNull(instanceData);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 74718c9..ada0047 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -20,14 +20,13 @@ package org.apache.slider.server.services.utility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperationsFactory;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.slider.server.services.curator.CuratorHelper;
-import org.apache.slider.server.services.registry.SliderRegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,21 +45,6 @@ public abstract class AbstractSliderLaunchedService extends
}
/**
- * Start the registration service
- * @return the instance
- * @throws BadConfigException
- */
- protected SliderRegistryService startRegistrationService()
- throws BadConfigException {
-
- String registryQuorum = lookupZKQuorum();
- String zkPath = getConfig().get(
- SliderXmlConfKeys.REGISTRY_PATH,
- SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
- return startSliderRegistrationService(registryQuorum, zkPath);
- }
-
- /**
* look up the registry quorum from the config
* @return the quorum string
* @throws BadConfigException if it is not there or invalid
@@ -86,36 +70,17 @@ public abstract class AbstractSliderLaunchedService extends
}
/**
- * Start the registration service
- * @param zkConnection
- * @param zkPath
- * @return
- */
- public SliderRegistryService startSliderRegistrationService(
- String zkConnection,
- String zkPath) {
- CuratorHelper curatorHelper =
- new CuratorHelper(getConfig(), zkConnection);
-
- //registry will start curator as well as the binder, in the correct order
- SliderRegistryService registryBinderService =
- curatorHelper.createRegistryBinderService(zkPath);
- deployChildService(registryBinderService);
- return registryBinderService;
- }
-
- /**
* Create, adopt ,and start the YARN registration service
* @return the registry operations service, already deployed as a child
* of the AbstractSliderLaunchedService instance.
*/
- public RegistryOperationsService startRegistryOperationsService()
+ public RegistryOperations startRegistryOperationsService()
throws BadConfigException {
// push back the slider registry entry if needed
String quorum = lookupZKQuorum();
getConfig().set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, quorum);
- RegistryOperationsService registryWriterService =
+ RegistryOperations registryWriterService =
createRegistryOperationsInstance();
deployChildService(registryWriterService);
return registryWriterService;
@@ -126,8 +91,8 @@ public abstract class AbstractSliderLaunchedService extends
* subclasses to instantiate a subclass service
* @return an instance to match to the lifecycle of this service
*/
- protected RegistryOperationsService createRegistryOperationsInstance() {
- return new RegistryOperationsService("YarnRegistry");
+ protected RegistryOperations createRegistryOperationsInstance() {
+ return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig());
}
/**
@@ -139,9 +104,8 @@ public abstract class AbstractSliderLaunchedService extends
protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
if (isUnset(value)) {
- throw new BadCommandArgumentsException("Required argument "
- + argname
- + " missing");
+ throw new BadCommandArgumentsException(
+ "Required argument " + argname + " missing");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/44e61659/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
index 65ae420..a08cfaf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java
@@ -18,9 +18,10 @@
package org.apache.slider.server.services.yarnregistry;
-import org.apache.hadoop.yarn.registry.client.binding.BindingUtils;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils;
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
+
import org.apache.hadoop.yarn.registry.client.api.CreateFlags;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
@@ -28,14 +29,15 @@ import java.io.IOException;
public class YarnRegistryViewForProviders {
- private final RegistryOperationsService registryOperations;
+ private final RegistryOperations registryOperations;
private final String user;
private final String sliderServiceclass;
private final String instanceName;
+ private ServiceRecord selfRegistration;
- public YarnRegistryViewForProviders(RegistryOperationsService registryOperations,
+ public YarnRegistryViewForProviders(RegistryOperations registryOperations,
String user, String sliderServiceclass, String instanceName) {
this.registryOperations = registryOperations;
this.user = user;
@@ -55,10 +57,18 @@ public class YarnRegistryViewForProviders {
return instanceName;
}
- public RegistryOperationsService getRegistryOperationsService() {
+ public RegistryOperations getRegistryOperations() {
return registryOperations;
}
+ public ServiceRecord getSelfRegistration() {
+ return selfRegistration;
+ }
+
+ public void setSelfRegistration(ServiceRecord selfRegistration) {
+ this.selfRegistration = selfRegistration;
+ }
+
/**
* Add a component under the slider name/entry
* @param componentName component name
@@ -70,8 +80,7 @@ public class YarnRegistryViewForProviders {
IOException {
putComponent(sliderServiceclass, instanceName,
componentName,
- record
- );
+ record);
}
/**
@@ -85,11 +94,10 @@ public class YarnRegistryViewForProviders {
String serviceName,
String componentName,
ServiceRecord record) throws IOException {
- String path = BindingUtils.componentPath(
+ String path = RegistryOperationUtils.componentPath(
user, serviceClass, serviceName, componentName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
- registryOperations.create(path, record,
- CreateFlags.OVERWRITE);
+ registryOperations.create(path, record, CreateFlags.OVERWRITE);
}
@@ -106,7 +114,7 @@ public class YarnRegistryViewForProviders {
ServiceRecord record) throws IOException {
- String path = BindingUtils.servicePath(
+ String path = RegistryOperationUtils.servicePath(
username, serviceClass, serviceName);
registryOperations.mknode(RegistryPathUtils.parentOf(path), true);
registryOperations.create(path, record, CreateFlags.OVERWRITE);
@@ -115,7 +123,7 @@ public class YarnRegistryViewForProviders {
public void rmComponent(String componentName) throws IOException {
- String path = BindingUtils.componentPath(
+ String path = RegistryOperationUtils.componentPath(
user, sliderServiceclass, instanceName,
componentName);
registryOperations.delete(path, false);
[12/50] [abbrv] git commit: SLIDER-149 revert an accidentall deletion
Posted by st...@apache.org.
SLIDER-149 revert an accidentall deletion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/373f0c4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/373f0c4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/373f0c4d
Branch: refs/heads/develop
Commit: 373f0c4d0ece943d400787d18ab87bfdd7d1f8d3
Parents: 3ed6b2d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 17 22:02:28 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 17 22:02:28 2014 +0100
----------------------------------------------------------------------
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/373f0c4d/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index d80ee62..2b4da86 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -248,6 +248,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
noOfNodeManagers,
numLocalDirs,
numLogDirs)
+ miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
createMiniHDFSCluster(name, conf)
[28/50] [abbrv] git commit: SLIDER-149 purge RegisteredEndpoint code,
and so all traces of the original registry
Posted by st...@apache.org.
SLIDER-149 purge RegisteredEndpoint code, and so all traces of the original registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/65e7684d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/65e7684d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/65e7684d
Branch: refs/heads/develop
Commit: 65e7684de15c6340caff0381ceb99f2a12ca3f72
Parents: 46fb5db
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 26 14:44:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 26 14:44:38 2014 +0100
----------------------------------------------------------------------
.../core/registry/info/RegisteredEndpoint.java | 180 -------------------
.../providers/agent/AgentProviderService.java | 2 +-
.../server/appmaster/state/RoleInstance.java | 29 +--
3 files changed, 16 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/65e7684d/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
deleted file mode 100644
index 2351e06..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry.info;
-
-import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
-import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@Deprecated
-public class RegisteredEndpoint {
-
- // standard types
-
- /**
- * URL: {@value}
- */
- public static final String TYPE_URL = AddressTypes.ADDRESS_URI;
-
-
- /**
- * "hostname:port" pair: {@value}
- */
- public static final String TYPE_INETADDRESS = AddressTypes.ADDRESS_HOSTNAME_AND_PORT;
-
- /**
- * simple path string: {@value}
- */
- public static final String TYPE_PATH = AddressTypes.ADDRESS_PATH;
-
- // standard protocols
- /**
- * Classic "Writable" Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_RPC = ProtocolTypes.PROTOCOL_HADOOP_IPC;
-
- /**
- * Protocol buffer based Hadoop IPC: {@value}
- */
- public static final String PROTOCOL_HADOOP_PROTOBUF = ProtocolTypes.PROTOCOL_HADOOP_IPC_PROTOBUF;
-
- /**
- * The address -format is driven by the type entry
- */
- public String address;
-
- /**
- * Protocol
- */
- public String protocol = "";
-
- public String type = "";
-
- /**
- * Human readable type
- */
- public String description = "";
-
- public RegisteredEndpoint() {
- }
-
- public RegisteredEndpoint(String address,
- String protocol,
- String type,
- String description) {
- this.address = address;
- this.protocol = protocol;
- this.type = type;
- this.description = description;
- }
-
- /**
- * Build an endpoint instance from a URI, extracting
- * the protocol from it
- * @param uri URI to set the value to
- * @param description description
- */
- public RegisteredEndpoint(URI uri,
- String description) {
-
- this.address = uri.toString();
- this.protocol = uri.getScheme();
- this.type = TYPE_URL;
- this.description = description;
- }
- /**
- * Build an endpoint instance from a URI, extracting
- * the protocol from it
- * @param uri URI to set the value to
- * @param description description
- */
- public RegisteredEndpoint(InetSocketAddress address,
- String protocol,
- String description) {
-
- this.address = address.toString();
- this.protocol = protocol;
- this.type = TYPE_INETADDRESS;
- this.description = description;
- }
-
- /**
- * Build an endpoint instance from a URL, extracting
- * the protocol from it
- * @param url URL to set the value to
- * @param description description
- */
- public RegisteredEndpoint(URL url,
- String description) throws URISyntaxException {
- this(url.toURI(), description);
- }
-
- /**
- * Get the value as a URL
- * @return URL of the value -if the value type is URL
- * @throws SliderException if the value is of the wrong type, or unparsable
- */
- public URL asURL() throws SliderException {
- verifyEndpointType(TYPE_URL);
- try {
- return new URL(address);
- } catch (MalformedURLException e) {
- throw new SliderException(-1, e,
- "could not create a URL from %s : %s", address, e.toString());
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder();
- if (TYPE_URL.equals(type)) {
- sb.append(address);
- } else {
- sb.append("protocol='").append(protocol).append('\'');
- sb.append(" address='").append(address).append('\'');
- sb.append(" type='").append(type).append('\'');
- }
- sb.append(" -- \"").append(description).append('"');
- return sb.toString();
- }
-
-
- /**
- * Verify that an endpoint is of the desired type
- * @param desiredType desired type
- * @throws SliderException if it is not
- */
- public void verifyEndpointType(String desiredType) throws SliderException {
- if (!type.equals(desiredType)) {
- throw new SliderException(-1, "Body of endpoint is of type %s and not %s",
- type, desiredType);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/65e7684d/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 5de08dc..abca155 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -595,7 +595,7 @@ public class AgentProviderService extends AbstractProviderService implements
this.getAllocatedPorts(containerId).put(portname, portNo);
if (instance != null) {
try {
- instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
+ instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
} catch (NumberFormatException e) {
log.warn("Failed to parse {}: {}", portNo, e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/65e7684d/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index e373843..df5e47e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -22,12 +22,13 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.types.Endpoint;
+import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -90,8 +91,8 @@ public final class RoleInstance implements Cloneable {
/**
* A list of registered endpoints.
*/
- private List<RegisteredEndpoint> endpoints =
- new ArrayList<RegisteredEndpoint>(2);
+ private List<Endpoint> endpoints =
+ new ArrayList<Endpoint>(2);
public RoleInstance(Container container) {
Preconditions.checkNotNull(container, "Null container");
@@ -194,7 +195,7 @@ public final class RoleInstance implements Cloneable {
public Object clone() throws CloneNotSupportedException {
RoleInstance cloned = (RoleInstance) super.clone();
// clone the endpoint list, but not the values
- cloned.endpoints = new ArrayList<RegisteredEndpoint>(this.endpoints);
+ cloned.endpoints = new ArrayList<Endpoint>(this.endpoints);
return cloned;
}
@@ -202,15 +203,15 @@ public final class RoleInstance implements Cloneable {
* Get the list of endpoints.
* @return the endpoint list.
*/
- public List<RegisteredEndpoint> getEndpoints() {
+ public List<Endpoint> getEndpoints() {
return endpoints;
}
/**
* Add an endpoint registration
- * @param endpoint
+ * @param endpoint endpoint (non-null)
*/
- public void addEndpoint(RegisteredEndpoint endpoint) {
+ public void addEndpoint(Endpoint endpoint) {
Preconditions.checkArgument(endpoint != null);
endpoints.add(endpoint);
}
@@ -218,13 +219,13 @@ public final class RoleInstance implements Cloneable {
/**
* Register a port endpoint as an inet-addr formatted endpoint, using the
* hostname as the first part of the address
- * @param port
- * @param protocol
- * @param text
+ * @param port port port
+ * @param api API API name
*/
- public void registerPortEndpoint(int port, String protocol, String text) {
- InetSocketAddress addr = new InetSocketAddress(host, port);
- RegisteredEndpoint epr = new RegisteredEndpoint(addr, protocol, text);
+ public void registerPortEndpoint(int port, String api) {
+ Endpoint epr =
+ RegistryTypeUtils.inetAddrEndpoint(api,
+ ProtocolTypes.PROTOCOL_TCP, host, port);
addEndpoint(epr);
}
[42/50] [abbrv] git commit: SLIDER-365 "slider resolve" command
Posted by st...@apache.org.
SLIDER-365 "slider resolve" command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/032dacf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/032dacf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/032dacf4
Branch: refs/heads/develop
Commit: 032dacf4cf1757302fffc59782572f574d185cdc
Parents: c09205f
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 17:03:21 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 17:03:21 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 301 +++++++------------
.../apache/slider/client/SliderClientAPI.java | 249 +++++++++++++++
.../slider/common/params/ActionResolveArgs.java | 5 -
.../apache/slider/common/params/ClientArgs.java | 4 +-
.../apache/slider/common/tools/SliderUtils.java | 30 +-
.../TestStandaloneYarnRegistryAM.groovy | 40 ++-
6 files changed, 418 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 185e77d..c5022ee 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -140,6 +141,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -159,7 +161,7 @@ import static org.apache.slider.common.params.SliderActions.*;
*/
public class SliderClient extends AbstractSliderLaunchedService implements RunService,
- SliderExitCodes, SliderKeys, ErrorStrings {
+ SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
private ClientArgs serviceArgs;
@@ -197,6 +199,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new YarnConfiguration();
}
+ /**
+ * This is called <i>Before serviceInit is called</i>
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the post-binding configuration to pass to the <code>init()</code>
+ * operation.
+ * @throws Exception
+ */
@Override
public Configuration bindArgs(Configuration config, String... args) throws Exception {
config = super.bindArgs(config, args);
@@ -218,9 +230,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-
+ AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+ if (coreAction.getHadoopServicesRequired()) {
+ initHadoopBinding();
+ }
super.serviceInit(conf);
-
}
/**
@@ -308,11 +322,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
-
- AbstractActionArgs coreAction = serviceArgs.getCoreAction();
- if (coreAction.getHadoopServicesRequired()) {
- initHadoopBinding();
- }
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -358,9 +368,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionStatusArgs());
} else if (ACTION_UPDATE.equals(action)) {
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
-
} else if (ACTION_VERSION.equals(action)) {
-
exitCode = actionVersion();
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
@@ -490,11 +498,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return client;
}
- /**
- * Destroy a cluster. There's two race conditions here
- * #1 the cluster is started between verifying that there are no live
- * clusters of that name.
- */
+ @Override
public int actionDestroy(String clustername) throws YarnException,
IOException {
// verify that a live cluster isn't there
@@ -536,11 +540,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * AM to commit an asynchronous suicide
- */
+ @Override
public int actionAmSuicide(String clustername,
- ActionAMSuicideArgs args) throws
+ ActionAMSuicideArgs args) throws
YarnException,
IOException {
SliderClusterOperations clusterOperations =
@@ -549,13 +551,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Get the provider for this cluster
- * @param provider the name of the provider
- * @return the provider instance
- * @throws SliderException problems building the provider
- */
- private AbstractClientProvider createClientProvider(String provider)
+ @Override
+ public AbstractClientProvider createClientProvider(String provider)
throws SliderException {
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory(provider);
@@ -622,17 +619,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- /**
- * Build up the cluster specification/directory
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to build the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
+ @Override
public int actionBuild(String clustername,
- AbstractClusterBuildingActionArgs buildInfo) throws
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException,
IOException {
@@ -640,24 +629,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Upload application package to user home directory
- *
- * @param installPkgInfo the arguments needed to build the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
+ @Override
public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
YarnException,
IOException {
Path srcFile = null;
- if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) {
+ if (StringUtils.isEmpty(installPkgInfo.name )) {
throw new BadCommandArgumentsException("A valid application type name is required (e.g. HBASE).");
}
- if (null == installPkgInfo.packageURI || installPkgInfo.packageURI.length() == 0) {
+ if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
throw new BadCommandArgumentsException("A valid application package location required.");
} else {
File pkgFile = new File(installPkgInfo.packageURI);
@@ -684,15 +666,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Update the cluster specification
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to update the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- */
- public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo) throws
+ @Override
+ public int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException, IOException {
buildInstanceDefinition(clustername, buildInfo, true, true);
return EXIT_SUCCESS;
@@ -1011,10 +987,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
*
- * @param clustername
- * @param clusterDirectory
- * @param instanceDefinition
- * @param debugAM
+ * @param clustername name of the cluster
+ * @param clusterDirectory cluster dir
+ * @param instanceDefinition the instance definition
+ * @param debugAM enable debug AM options
* @return the launched application
* @throws YarnException
* @throws IOException
@@ -1579,23 +1555,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return launchedApplication.monitorAppToState(desiredState, duration);
}
- /**
- * Get the report of a this application
- * @return the app report or null if it could not be found.
- * @throws IOException
- * @throws YarnException
- */
+ @Override
public ApplicationReport getApplicationReport() throws
IOException,
YarnException {
return getApplicationReport(applicationId);
}
- /**
- * Kill the submitted application via YARN
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public boolean forceKillApplication(String reason)
throws YarnException, IOException {
if (applicationId != null) {
@@ -1616,10 +1583,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return YarnAppListClient.listInstances();
}
- /**
- * Implement the list action: list all nodes
- * @return exit code of 0 if a list was created
- */
+ @Override
@VisibleForTesting
public int actionList(String clustername) throws IOException, YarnException {
verifyBindingsDefined();
@@ -1657,10 +1621,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info(SliderUtils.appReportToString(report, "\n"));
}
- /**
- * Implement the islive action: probe for a cluster of the given name existing
- * @return exit code
- */
+ @Override
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
verifyBindingsDefined();
@@ -1682,11 +1643,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return flex(name, roleInstances);
}
- /**
- * Test for a cluster existing probe for a cluster of the given name existing
- * in the filesystem. If the live param is set, it must be a live cluster
- * @return exit code
- */
+ @Override
@VisibleForTesting
public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
verifyBindingsDefined();
@@ -1727,18 +1684,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
- /**
- * Kill a specific container of the cluster
- * @param name cluster name
- * @param args arguments
- * @return exit code
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public int actionKillContainer(String name,
- ActionKillContainerArgs args) throws
- YarnException,
- IOException {
+ ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
if (SliderUtils.isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
@@ -1755,14 +1703,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Echo operation (not currently wired up to command line)
- * @param name cluster name
- * @param args arguments
- * @return the echoed text
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public String actionEcho(String name, ActionEchoArgs args) throws
YarnException,
IOException {
@@ -1790,15 +1731,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
- private ApplicationReport findInstance(String appname) throws
- YarnException,
- IOException {
+ private ApplicationReport findInstance(String appname)
+ throws YarnException, IOException {
return YarnAppListClient.findInstance(appname);
}
- private RunningApplication findApplication(String appname) throws
- YarnException,
- IOException {
+ private RunningApplication findApplication(String appname)
+ throws YarnException, IOException {
ApplicationReport applicationReport = findInstance(appname);
return applicationReport != null ? new RunningApplication(yarnClient, applicationReport): null;
@@ -1829,9 +1768,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException
* @throws IOException
*/
- private SliderClusterProtocol connect(ApplicationReport app) throws
- YarnException,
- IOException {
+ private SliderClusterProtocol connect(ApplicationReport app)
+ throws YarnException, IOException {
try {
return RpcBinder.getProxy(getConfig(),
@@ -1846,15 +1784,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- /**
- * Status operation
- *
- * @param clustername cluster name
- * @param statusArgs status arguments
- * @return 0 -for success, else an exception is thrown
- * @throws YarnException
- * @throws IOException
- */
+ @Override
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
YarnException,
@@ -1872,26 +1802,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Version Details
- * @return exit code
- */
+ @Override
public int actionVersion() {
SliderVersionInfo.loadAndPrintVersionInfo(log);
return EXIT_SUCCESS;
}
- /**
- * Stop the cluster
- *
- * @param clustername cluster name
- * @param freezeArgs arguments to the stop
- * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
- */
+ @Override
public int actionFreeze(String clustername,
- ActionFreezeArgs freezeArgs) throws
- YarnException,
- IOException {
+ ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
@@ -2004,9 +1923,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@SuppressWarnings(
{"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs) throws
- YarnException,
- IOException {
+ public int actionGetConf(String clustername, ActionGetConfArgs confArgs)
+ throws YarnException, IOException {
File outfile = null;
if (confArgs.getOutput() != null) {
@@ -2067,9 +1985,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Restore a cluster
- */
+ @Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
SliderUtils.validateClusterName(clustername);
// see if it is actually running and bail out;
@@ -2089,10 +2005,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException
* @throws IOException
*/
- public int flex(String clustername,
- Map<String, Integer> roleInstances) throws
- YarnException,
- IOException {
+ public int flex(String clustername, Map<String, Integer> roleInstances)
+ throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
@@ -2382,58 +2296,67 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
- /**
- * Registry operation
- *
- * @param args registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- public int actionResolve(ActionResolveArgs args) throws
- YarnException,
- IOException {
- // as this is also a test entry point, validate
+ @Override
+ public int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException {
+ // as this is an API entry point, validate
// the arguments
args.validate();
RegistryOperations operations = getRegistryOperations();
String path = args.path;
Collection<ServiceRecord> serviceRecords;
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
+ File outputPath = args.out;
try {
if (args.list) {
Map<String, ServiceRecord> recordMap =
listServiceRecords(operations, path);
serviceRecords = recordMap.values();
// list records out
+ StringBuilder builder = new StringBuilder(1024);
+ for (Entry<String, ServiceRecord> recordEntry : recordMap
+ .entrySet()) {
+ ServiceRecord instance = recordEntry.getValue();
+ builder.append("\"").append(recordEntry.getKey()).append("\":\n");
+ builder.append(serviceRecordMarshal.toJson(instance));
+ builder.append("}\n");
+ }
+ String records = builder.toString();
+ if (outputPath == null) {
+ print(records);
+ } else {
+ SliderUtils.write(outputPath, records.getBytes("UTF-8"), false);
+ }
} else {
+ // resolve single entry
ServiceRecord instance = resolve(path);
serviceRecords = new ArrayList<ServiceRecord>(1);
serviceRecords.add(instance);
- // list or save records
+ // write out JSON content
+ if (outputPath != null) {
+ byte[] data = serviceRecordMarshal.toBytes(instance);
+ SliderUtils.write(outputPath, data, false);
+ } else {
+ // print to the console
+ print(serviceRecordMarshal.toJson(instance));
+ }
}
// JDK7
- } catch (FileNotFoundException e) {
- log.info("{}", e);
- log.debug("{}", e, e);
- return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e);
- log.debug("{}", e, e);
+ // no record at this path
+ return EXIT_NOT_FOUND;
+ } catch (NoRecordException e) {
return EXIT_NOT_FOUND;
+ } catch (InvalidRecordException e) {
+ // it is not a record
+ log.error("{}", e);
+ log.debug("{}", e, e);
+ return EXIT_EXCEPTION_THROWN;
}
return EXIT_SUCCESS;
}
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
+ @Override
public int actionRegistry(ActionRegistryArgs registryArgs) throws
YarnException,
IOException {
@@ -2514,22 +2437,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return serviceRecords;
}
- /**
- * diagnostic operation
- *
- * @param clusterName
- * application name
- * @param diagosticArgs
- * diagnostic Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just
- * failures to retrieve information (e.g. no application name
- * specified)
- * @throws YarnException
- * YARN problems
- * @throws IOException
- * Network or other problems
- */
- public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ @Override
+ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
try {
if (diagnosticArgs.client) {
actionDiagnosticClient();
@@ -2862,9 +2771,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * write out the config
- * @param published
- * @param registryArgs
+ * write out the config. If a destination is provided and that dir is a
+ * directory, the entry is written to it with the name provided + extension,
+ * else it is printed to standard out.
+ * @param published published config
+ * @param registryArgs registry Arguments
* @throws BadCommandArgumentsException
* @throws IOException
*/
@@ -3007,11 +2918,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return registryOperations;
}
- /**
- * Get the registry binding. As this may start the registry, it can take time
- * and fail
- * @return the registry
- */
+ @Override
public RegistryOperations getRegistryOperations()
throws SliderException, IOException {
return maybeStartYarnRegistry();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
new file mode 100644
index 0000000..cacf962
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+
+import java.io.IOException;
+
+/**
+ * Interface of those method calls in the slider API that are intended
+ * for direct public invocation.
+ * <p>
+ * Stability: evolving
+ */
+public interface SliderClientAPI extends Service {
+ /**
+ * Destroy a cluster. There's two race conditions here
+ * #1 the cluster is started between verifying that there are no live
+ * clusters of that name.
+ */
+ int actionDestroy(String clustername) throws YarnException,
+ IOException;
+
+ /**
+ * AM to commit an asynchronous suicide
+ */
+ int actionAmSuicide(String clustername,
+ ActionAMSuicideArgs args) throws YarnException, IOException;
+
+ /**
+ * Get the provider for this cluster
+ * @param provider the name of the provider
+ * @return the provider instance
+ * @throws SliderException problems building the provider
+ */
+ AbstractClientProvider createClientProvider(String provider)
+ throws SliderException;
+
+ /**
+ * Build up the cluster specification/directory
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionBuild(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws YarnException, IOException;
+
+ /**
+ * Upload application package to user home directory
+ *
+ * @param installPkgInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionInstallPkg(ActionInstallPackageArgs installPkgInfo)
+ throws YarnException, IOException;
+
+ /**
+ * Update the cluster specification
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to update the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ */
+ int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo)
+ throws YarnException, IOException;
+ /**
+ * Get the report of a this application
+ * @return the app report or null if it could not be found.
+ * @throws IOException
+ * @throws YarnException
+ */
+ ApplicationReport getApplicationReport()
+ throws IOException, YarnException;
+
+ /**
+ * Kill the submitted application via YARN
+ * @throws YarnException
+ * @throws IOException
+ */
+ boolean forceKillApplication(String reason)
+ throws YarnException, IOException;
+
+ /**
+ * Implement the list action: list all nodes
+ * @return exit code of 0 if a list was created
+ */
+ @VisibleForTesting
+ int actionList(String clustername) throws IOException, YarnException;
+
+ /**
+ * Implement the islive action: probe for a cluster of the given name existing
+ * @return exit code
+ */
+ @VisibleForTesting
+ int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
+
+ /**
+ * Test for a cluster existing probe for a cluster of the given name existing
+ * in the filesystem. If the live param is set, it must be a live cluster
+ * @return exit code
+ */
+ @VisibleForTesting
+ int actionExists(String name, boolean checkLive) throws YarnException, IOException;
+
+ /**
+ * Kill a specific container of the cluster
+ * @param name cluster name
+ * @param args arguments
+ * @return exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ int actionKillContainer(String name, ActionKillContainerArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Echo operation (not currently wired up to command line)
+ * @param name cluster name
+ * @param args arguments
+ * @return the echoed text
+ * @throws YarnException
+ * @throws IOException
+ */
+ String actionEcho(String name, ActionEchoArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Status operation
+ *
+ * @param clustername cluster name
+ * @param statusArgs status arguments
+ * @return 0 -for success, else an exception is thrown
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ int actionStatus(String clustername, ActionStatusArgs statusArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Version Details
+ * @return exit code
+ */
+ int actionVersion();
+
+ /**
+ * Stop the cluster
+ *
+ * @param clustername cluster name
+ * @param freezeArgs arguments to the stop
+ * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
+ */
+ int actionFreeze(String clustername, ActionFreezeArgs freezeArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Restore a cluster
+ */
+ int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param args registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param registryArgs registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionRegistry(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException;
+
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs);
+
+ /**
+ * Get the registry binding. As this may start the registry, it can take time
+ * and fail
+ * @return the registry
+ */
+ RegistryOperations getRegistryOperations()
+ throws SliderException, IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
index 9b6a5f1..cf5611c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -83,10 +83,5 @@ public class ActionResolveArgs extends AbstractActionArgs {
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")
public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 0fffffb..16363d2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -191,7 +191,7 @@ public class ClientArgs extends CommonArgs {
* Look at the chosen action and bind it as the core action for the operation.
* In theory this could be done by introspecting on the list of actions and
* choosing it without the switch statement. In practise this switch, while
- * verbose, is easier to debug.
+ * verbose, is easier to debug. And in JDK7, much simpler.
* @throws SliderException bad argument or similar
*/
@Override
@@ -206,7 +206,7 @@ public class ClientArgs extends CommonArgs {
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
- }else if (SliderActions.ACTION_FREEZE.equals(action)) {
+ } else if (SliderActions.ACTION_FREEZE.equals(action)) {
bindCoreAction(actionFreezeArgs);
} else if (SliderActions.ACTION_THAW.equals(action)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 9c93753..72aa4fe 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
@@ -1664,8 +1665,8 @@ public final class SliderUtils {
/**
* Validate an executable
- * @param program
- * @param exe
+ * @param program program name for errors
+ * @param exe program to look at
* @throws IOException
*/
public static void validateExe(String program, File exe) throws IOException {
@@ -1676,6 +1677,29 @@ public final class SliderUtils {
}
}
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @param createParent flag to indicate that the parent dir should
+ * be created
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data, boolean createParent)
+ throws IOException {
+ File parentDir = outfile.getParentFile();
+ if (createParent) {
+ parentDir.mkdirs();
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ FileOutputStream out = new FileOutputStream(outfile);
+ try {
+ out.write(data);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+
+ }
/**
* Execute a command for a test operation
@@ -1683,7 +1707,7 @@ public final class SliderUtils {
* @param status status code expected
* @param timeoutMillis timeout in millis for process to finish
* @param logger
- *@param outputString optional string to grep for (must not span a line)
+ * @param outputString optional string to grep for (must not span a line)
* @param commands commands @return the process
* @throws IOException on any failure.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 1f65d2f..97e995d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -29,6 +29,7 @@ import org.apache.hadoop.registry.client.impl.RegistryOperationsClient
import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import static org.apache.hadoop.registry.client.binding.RegistryUtils.*
@@ -109,9 +110,6 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
ClusterNode master = nodes[0]
assert master.role == SliderKeys.COMPONENT_AM
-
-
-
String username = client.username
def yarnRegistryClient = client.yarnAppListClient
describe("list of all applications")
@@ -174,12 +172,47 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
+ // use the resolve operation
+ describe "resolve CLI action"
+ ActionResolveArgs resolveArgs = new ActionResolveArgs()
+ resolveArgs.path = recordsPath;
+ resolveArgs.list = true;
+ // to stdout
+ client.actionResolve(resolveArgs)
+ // to a file
+ File destFile = new File("target/resolve.json")
+ destFile.delete()
+ resolveArgs.out = destFile
+ client.actionResolve(resolveArgs)
+ assert destFile.exists()
+ destFile.delete()
+
+ // look at a single record
+ resolveArgs.out = null;
+ resolveArgs.list = false;
+ resolveArgs.path = recordsPath +"/"+ clustername
+ // to stdout
+ client.actionResolve(resolveArgs)
+ resolveArgs.out = destFile
+ client.actionResolve(resolveArgs)
+ assert destFile.exists()
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal()
+ def recordFromFile = serviceRecordMarshal.fromFile(destFile)
+ assert recordFromFile[YarnRegistryAttributes.YARN_ID] ==
+ serviceRecord[YarnRegistryAttributes.YARN_ID]
+ assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] ==
+ serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
+
+
+
// hit the registry web page
def registryEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.REGISTRY_REST_API)
assert registryEndpoint != null
def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+
+ // Look at the Registry WADL
describe("Registry WADL @ $registryURL")
def publisherEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_REST_API)
@@ -393,6 +426,5 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
-
}
}
[05/50] [abbrv] git commit: Merge branch 'feature/SLIDER-86-windows'
into feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-86-windows' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f6bbb8b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f6bbb8b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f6bbb8b8
Branch: refs/heads/develop
Commit: f6bbb8b8574c111acda2cc51918b00ed85440d60
Parents: 5784c4a 4a0b783
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 10 16:43:51 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 10 16:43:51 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/README.txt | 17 +-
app-packages/accumulo/pom.xml | 90 +++++-
app-packages/accumulo/src/assembly/accumulo.xml | 2 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 7 +
.../accumulo/AccumuloMonitorSSLIT.groovy | 7 +-
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 7 +-
.../accumulo/src/test/resources/appConfig.json | 55 ++++
.../src/test/resources/appConfig_kerberos.json | 8 +-
.../test/resources/appConfig_monitor_ssl.json | 4 +-
.../appConfig_monitor_ssl_kerberos.json | 57 ++++
.../src/test/resources/appConfig_ssl.json | 4 +-
.../test/resources/appConfig_ssl_kerberos.json | 59 ++++
.../command-logger/application-pkg/pom.xml | 38 ++-
app-packages/command-logger/slider-pkg/pom.xml | 38 ++-
app-packages/hbase/README.txt | 43 ++-
app-packages/hbase/appConfig.json | 4 +-
app-packages/hbase/metainfo.xml | 4 +-
app-packages/hbase/pom.xml | 317 +++++++++++--------
app-packages/hbase/src/assembly/hbase.xml | 16 +-
app-packages/storm/README.txt | 9 +-
app-packages/storm/appConfig.json | 8 +-
app-packages/storm/metainfo.xml | 2 +-
app-packages/storm/package/scripts/params.py | 8 +-
app-packages/storm/package/scripts/service.py | 47 ++-
app-packages/storm/package/scripts/storm.py | 9 +-
.../storm/package/scripts/yaml_config.py | 10 +-
.../storm/package/templates/storm_jaas.conf.j2 | 11 +-
app-packages/storm/resources.json | 2 +-
pom.xml | 78 ++---
slider-agent/pom.xml | 62 ++--
slider-assembly/pom.xml | 23 +-
slider-assembly/src/main/scripts/slider | 2 +
slider-assembly/src/main/scripts/slider.py | 119 ++++---
slider-core/pom.xml | 102 ++----
.../org/apache/slider/client/SliderClient.java | 92 +++++-
.../org/apache/slider/common/SliderKeys.java | 2 +
.../common/params/AbstractActionArgs.java | 3 +-
.../common/params/ActionInstallPackageArgs.java | 58 ++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 12 +-
.../slider/common/params/SliderActions.java | 2 +
.../slider/common/tools/CoreFileSystem.java | 12 +
.../conf/TemplateInputPropertiesValidator.java | 2 +-
.../slider/core/launch/AppMasterLauncher.java | 6 +-
.../providers/agent/AgentProviderService.java | 15 +-
.../security/FsDelegationTokenManager.java | 38 ++-
.../actions/TestActionInstallPackage.groovy | 157 +++++++++
.../TestFsDelegationTokenManager.groovy | 3 +-
.../agent/TestAgentProviderService.java | 10 +-
slider-funtest/pom.xml | 48 +--
.../framework/AgentCommandTestBase.groovy | 2 +-
.../funtest/framework/CommandTestBase.groovy | 7 +-
.../funtest/framework/FileUploader.groovy | 1 +
.../funtest/framework/FuntestProperties.groovy | 1 +
.../slider/funtest/framework/SliderShell.groovy | 85 ++++-
.../slider/funtest/basic/SyspropsIT.groovy | 50 +++
.../hbase/slider-hbase-provider/pom.xml | 49 +--
src/test/clusters/script.md | 2 +-
58 files changed, 1394 insertions(+), 533 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ed3bc0e,ef0e63c..350d20d
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -213,16 -196,10 +214,13 @@@ public class SliderClient extends Abstr
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
- //create the YARN client
- yarnClient = new SliderYarnClientImpl();
- addService(yarnClient);
-
super.serviceInit(conf);
-
+
+ //here the superclass is inited; getConfig returns a non-null value
+ sliderFileSystem = new SliderFileSystem(getConfig());
+ YarnAppListClient =
+ new YarnAppListClient(yarnClient, getUsername(), getConfig());
}
/**
@@@ -370,6 -345,31 +366,34 @@@
return exitCode;
}
+
+ /**
+ * Perform everything needed to init the hadoop binding.
+ * This assumes that the service is already in inited or started state
+ * @throws IOException
+ * @throws SliderException
+ */
+ protected void initHadoopBinding() throws IOException, SliderException {
+ // validate the client
+ SliderUtils.validateSliderClientEnvironment(null);
+ //create the YARN client
+ yarnClient = new SliderYarnClientImpl();
+ yarnClient.init(getConfig());
+ if (getServiceState() == STATE.STARTED) {
+ yarnClient.start();
+ }
+ addService(yarnClient);
+ // create the filesystem
+ sliderFileSystem = new SliderFileSystem(getConfig());
+
+ // and the registry
++/*
+ YARNRegistryClient =
+ new YARNRegistryClient(yarnClient, getUsername(), getConfig());
++*/
++
+ }
+
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f6bbb8b8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
[06/50] [abbrv] git commit: Merge commit
'7c95d5b059045cc01fa9756308f50e01644abb2d' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge commit '7c95d5b059045cc01fa9756308f50e01644abb2d' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4eeed33a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4eeed33a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4eeed33a
Branch: refs/heads/develop
Commit: 4eeed33abd8623c094ea771e1d6cd2875c15c1fd
Parents: f6bbb8b 7c95d5b
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 11 10:37:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 11 10:37:11 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 13 ++++++-------
.../org/apache/slider/common/tools/SliderUtils.java | 4 +++-
.../abstracttests/AbstractTestBuildSetup.groovy | 3 +++
3 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eeed33a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
[31/50] [abbrv] git commit: SLIDER-149 sync with move to short-named
status
Posted by st...@apache.org.
SLIDER-149 sync with move to short-named status
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/62335bfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/62335bfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/62335bfd
Branch: refs/heads/develop
Commit: 62335bfd1a894db41016fb0aa14268c5e21737e2
Parents: 969b734
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 30 15:24:18 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 30 15:24:18 2014 -0700
----------------------------------------------------------------------
.../TestStandaloneYarnRegistryAM.groovy | 1 +
.../apache/slider/test/SliderTestUtils.groovy | 7 +++++
.../minicluster/live/TestTwoLiveClusters.groovy | 32 ++++++++++++--------
3 files changed, 28 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index d081676..0155232 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -154,6 +154,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
registryService,
+ recordsPath,
registryService.listFull(recordsPath))
def serviceRecords = recordMap.values();
dumpCollection(serviceRecords)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 9f8e850..fe9e587 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -713,6 +713,13 @@ class SliderTestUtils extends Assert {
log.info("number of entries: ${entries.length}")
entries.each { log.info(it.toString()) }
}
+
+ public static void dumpMap(Map map) {
+ map.entrySet().each { Map.Entry it ->
+ log.info("\"${it.key.toString()}\": \"${it.value.toString()}\"")
+ }
+ }
+
/**
* Get a time option in seconds if set, otherwise the default value (also in seconds).
* This operation picks up the time value as a system property if set -that
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/62335bfd/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
index a415524..01a67c7 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
@@ -20,11 +20,11 @@ package org.apache.slider.providers.hbase.minicluster.live
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
import org.apache.hadoop.yarn.registry.client.api.RegistryOperations
import org.apache.hadoop.yarn.registry.client.binding.RegistryOperationUtils
import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.HBaseKeys
@@ -87,16 +87,20 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
describe "service registry names"
RegistryOperations registry = cluster2Client.registryOperations
def home = RegistryOperationUtils.homePathForCurrentUser()
- def names = RegistryOperationUtils.listServiceRecords(registry,
- RegistryPathUtils.join(home, RegistryConstants.PATH_USER_SERVICES))
- def stats = registry.listFull(
- RegistryPathUtils.join(home, RegistryConstants.PATH_USER_SERVICES))
+ def userSliderInstancesPath = RegistryOperationUtils.serviceclassPath(
+ RegistryOperationUtils.currentUser(), SliderKeys.APP_TYPE)
+
+
+ def names = RegistryOperationUtils.listServiceRecords(registry,
+ userSliderInstancesPath)
+ dumpMap(names)
+
+ def stats = registry.listFull(userSliderInstancesPath)
dumpCollection(stats)
List<String> instanceIds = sliderClient.listRegisteredSliderInstances()
-
dumpRegistryInstanceIDs(instanceIds)
assert names.size() == 2
assert instanceIds.size() == 2
@@ -107,16 +111,20 @@ class TestTwoLiveClusters extends HBaseMiniClusterTestBase {
dumpRegistryInstances(instances)
assert instances.size() == 2
+
+ def hbaseServicePath = RegistryOperationUtils.serviceclassPath(
+ RegistryOperationUtils.currentUser(),
+ HBaseKeys.HBASE_SERVICE_TYPE)
Map<String, ServiceRecord> hbaseInstances =
RegistryOperationUtils.listServiceRecords(registry,
- RegistryOperationUtils.serviceclassPath(
- RegistryOperationUtils.currentUser(),
- HBaseKeys.HBASE_SERVICE_TYPE));
+ hbaseServicePath);
assert hbaseInstances.size() == 2
- def hbase1ServiceData = hbaseInstances[clustername1]
- def hbase2ServiceData = hbaseInstances[clustername2]
- assert !(hbase1ServiceData == hbase2ServiceData)
+ String clusterPath1 = RegistryPathUtils.join(hbaseServicePath, clustername1)
+ String clusterPath2 = RegistryPathUtils.join(hbaseServicePath, clustername2)
+ assert hbaseInstances[clusterPath1] != null
+ assert hbaseInstances[clusterPath2] != null
+ assert hbaseInstances[clusterPath1] != hbaseInstances[clusterPath2]
clusterActionFreeze(cluster2Client, clustername2, "stop cluster 2")
clusterActionFreeze(sliderClient, clustername1, "Stop cluster 1")
[17/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/db52ff56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/db52ff56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/db52ff56
Branch: refs/heads/develop
Commit: db52ff5600a5825a7f2699ea11e14f8b1ff12d3d
Parents: 821d306 937e0a7
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:06:17 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:06:17 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 8 +-
app-packages/accumulo/metainfo.xml | 4 +-
app-packages/accumulo/pom.xml | 50 +++-----
app-packages/accumulo/src/assembly/accumulo.xml | 1 -
.../apache/slider/accumulo/ProviderUtil.java | 25 +---
.../funtest/accumulo/AccumuloBasicIT.groovy | 30 ++++-
.../accumulo/AccumuloMonitorSSLIT.groovy | 18 ++-
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 19 +++-
.../accumulo/src/test/resources/appConfig.json | 55 ---------
.../src/test/resources/appConfig_kerberos.json | 57 ----------
.../test/resources/appConfig_monitor_ssl.json | 55 ---------
.../appConfig_monitor_ssl_kerberos.json | 57 ----------
.../src/test/resources/appConfig_ssl.json | 57 ----------
.../test/resources/appConfig_ssl_kerberos.json | 59 ----------
.../command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/hbase-win/README.txt | 29 +++--
app-packages/hbase-win/appConfig.json | 72 ++++++------
app-packages/hbase-win/metainfo.xml | 8 +-
...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 ---
app-packages/hbase-win/pom.xml | 94 +++++++++++++++
app-packages/hbase-win/src/assembly/hbase.xml | 68 +++++++++++
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/ganglia_metrics.json | 38 +++++++
app-packages/hbase/metainfo.xml | 10 +-
app-packages/hbase/resources.json | 2 +-
app-packages/memcached-win/appConfig.json | 2 +-
app-packages/memcached/appConfig.json | 2 +-
app-packages/storm/appConfig.json | 4 +-
app-packages/storm/metainfo.xml | 32 +++++-
app-packages/storm/package/scripts/params.py | 4 +
app-packages/storm/package/scripts/service.py | 40 ++++---
.../storm/package/scripts/status_params.py | 2 +
.../storm/package/templates/config.yaml.j2 | 9 ++
app-packages/storm/resources.json | 4 +
pom.xml | 23 ++++
slider-assembly/src/conf-hdp/slider-client.xml | 28 +++--
slider-assembly/src/conf-hdp/slider-env.sh | 26 +++++
slider-assembly/src/conf/slider-client.xml | 44 ++++----
slider-assembly/src/conf/slider-env.sh | 26 +++++
slider-assembly/src/main/scripts/slider | 9 +-
slider-assembly/src/main/scripts/slider.py | 44 ++++++--
.../org/apache/slider/api/InternalKeys.java | 4 +
.../org/apache/slider/client/SliderClient.java | 7 +-
.../common/SliderXMLConfKeysForTesting.java | 2 +
.../AbstractClusterBuildingActionArgs.java | 4 +
.../apache/slider/common/params/Arguments.java | 1 +
.../slider/core/build/InstanceBuilder.java | 16 +++
.../slider/core/persist/JsonSerDeser.java | 17 +++
.../providers/agent/AgentProviderService.java | 2 +-
.../test_command_log/appConfig.json | 4 +-
.../test_command_log/appConfig_fast_no_reg.json | 3 +-
.../test_command_log/appConfig_no_hb.json | 4 +-
.../resources_queue_labels.json | 17 +++
.../providers/agent/TestBuildBasicAgent.groovy | 39 ++++++-
slider-funtest/pom.xml | 1 +
.../framework/AgentCommandTestBase.groovy | 79 ++++++++-----
.../funtest/lifecycle/AMFailuresIT.groovy | 1 -
.../lifecycle/AgentClusterLifecycleIT.groovy | 1 -
.../funtest/lifecycle/AgentFailures2IT.groovy | 1 -
.../funtest/lifecycle/AgentFailuresIT.groovy | 1 -
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 1 -
.../AppsThroughAgentQueueAndLabelsIT.groovy | 113 +++++++++++++++++++
.../lifecycle/ClusterBuildDestroyIT.groovy | 1 -
.../clusters/remote/slider/slider-client.xml | 14 ++-
64 files changed, 852 insertions(+), 617 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/db52ff56/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/db52ff56/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/db52ff56/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/db52ff56/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
[47/50] [abbrv] git commit: SLIDER-149 test to work even without
YARN-2571 setup
Posted by st...@apache.org.
SLIDER-149 test to work even without YARN-2571 setup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7887fd3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7887fd3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7887fd3c
Branch: refs/heads/develop
Commit: 7887fd3c01b67414c764d163e03953ad3dfcfc94
Parents: 3a759d1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 14:41:05 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 14:41:05 2014 -0700
----------------------------------------------------------------------
.../agent/standalone/TestStandaloneYarnRegistryAM.groovy | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7887fd3c/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 83774d2..84dcc50 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -74,7 +75,11 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
// get local binding
def registryOperations = microZKCluster.registryOperations
- registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+ try {
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+ } catch (PathNotFoundException e) {
+ log.warn(" RM is not apparently running registry services: {}", e, e)
+ }
ServiceLauncher<SliderClient> launcher
launcher = createStandaloneAM(clustername, true, false)
[50/50] [abbrv] git commit: Merge branch
'feature/SLIDER-149_Support_a_YARN_service_registry' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-149_Support_a_YARN_service_registry' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2553f157
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2553f157
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2553f157
Branch: refs/heads/develop
Commit: 2553f15737e8ce55903896a7a49c20658080f2cf
Parents: 3b32b8f 851f447
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 8 15:52:16 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Oct 8 15:52:16 2014 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 17 +-
pom.xml | 6 +
slider-core/pom.xml | 16 +-
.../org/apache/slider/client/SliderClient.java | 624 ++++++++++---------
.../apache/slider/client/SliderClientAPI.java | 249 ++++++++
.../slider/client/SliderClusterOperations.java | 1 -
.../common/params/ActionRegistryArgs.java | 9 +-
.../slider/common/params/ActionResolveArgs.java | 87 +++
.../org/apache/slider/common/params/ArgOps.java | 2 +-
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 67 +-
.../slider/common/params/SliderActions.java | 1 +
.../slider/common/tools/CoreFileSystem.java | 31 +-
.../apache/slider/common/tools/SliderUtils.java | 44 +-
.../core/exceptions/ExceptionConverter.java | 16 +-
.../slider/core/launch/AbstractLauncher.java | 7 +-
.../core/launch/JavaCommandLineBuilder.java | 4 +-
.../slider/core/persist/JsonSerDeser.java | 12 +-
.../core/registry/YARNRegistryClient.java | 85 ---
.../slider/core/registry/YarnAppListClient.java | 93 +++
.../docstore/PublishedConfiguration.java | 4 +-
.../registry/info/CommonRegistryConstants.java | 25 -
.../registry/info/CustomRegistryConstants.java | 8 +
.../core/registry/info/RegisteredDocument.java | 33 -
.../core/registry/info/RegisteredEndpoint.java | 206 ------
.../core/registry/info/RegistryFields.java | 34 -
.../core/registry/info/RegistryNaming.java | 51 --
.../slider/core/registry/info/RegistryView.java | 41 --
.../core/registry/info/ServiceInstanceData.java | 139 -----
.../registry/retrieve/RegistryRetriever.java | 83 ++-
.../providers/AbstractProviderService.java | 59 +-
.../slider/providers/ProviderService.java | 21 +-
.../providers/agent/AgentProviderService.java | 46 +-
.../providers/agent/ComponentInstanceState.java | 1 -
.../slideram/SliderAMClientProvider.java | 6 -
.../slideram/SliderAMProviderService.java | 80 ++-
.../server/appmaster/SliderAppMaster.java | 208 ++++---
.../actions/RegisterComponentInstance.java | 9 +-
.../server/appmaster/state/RoleInstance.java | 29 +-
.../server/appmaster/web/SliderAMWebApp.java | 23 +-
.../slider/server/appmaster/web/WebAppApi.java | 4 +
.../server/appmaster/web/WebAppApiImpl.java | 17 +-
.../appmaster/web/rest/AMWebServices.java | 9 +
.../server/appmaster/web/rest/RestPaths.java | 4 +-
.../web/rest/registry/PathEntryResource.java | 45 ++
.../web/rest/registry/RegistryResource.java | 157 +++++
.../server/services/curator/CuratorHelper.java | 138 ----
.../server/services/curator/CuratorService.java | 98 ---
.../curator/CuratorServiceInstance.java | 77 ---
.../curator/CuratorServiceInstances.java | 41 --
.../server/services/curator/CuratorUriSpec.java | 57 --
.../services/curator/RegistryBinderService.java | 315 ----------
.../registry/RegistryDiscoveryContext.java | 36 --
.../registry/RegistryRestResources.java | 178 ------
.../registry/RegistryServiceConstants.java | 26 -
.../registry/RegistryViewForProviders.java | 48 --
.../registry/SliderRegistryService.java | 99 ---
.../utility/AbstractSliderLaunchedService.java | 84 +--
.../YarnRegistryViewForProviders.java | 196 ++++++
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 6 +-
.../standalone/TestStandaloneAgentAM.groovy | 24 +-
.../standalone/TestStandaloneRegistryAM.groovy | 362 -----------
.../TestStandaloneYarnRegistryAM.groovy | 431 +++++++++++++
.../registry/curator/TestLocalRegistry.groovy | 164 -----
.../curator/TestRegistryRestResources.groovy | 195 ------
.../curator/TestServiceInstanceSerDeser.groovy | 122 ----
.../model/mock/MockProviderService.groovy | 18 +-
.../rest/registry/PathEntryMarshalling.groovy | 28 +
.../registry/TestRegistryRestMarshalling.groovy | 48 ++
.../registry/TestRegistryRestResources.groovy | 215 +++++++
.../view/TestClusterSpecificationBlock.groovy | 7 +-
.../web/view/TestContainerStatsBlock.groovy | 7 +-
.../appmaster/web/view/TestIndexBlock.groovy | 8 +-
.../apache/slider/test/MicroZKCluster.groovy | 29 +-
.../apache/slider/test/SliderTestUtils.groovy | 31 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 1 -
.../test/YarnZKMiniClusterTestBase.groovy | 23 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 2 +-
.../publisher/TestAgentProviderService.java | 4 +-
slider-core/src/test/resources/log4j.properties | 3 +-
.../appmaster/web/rest/registry/sample.json | 75 +++
.../funtest/FunctionalHBaseClusterIT.groovy | 7 +-
.../slider/providers/hbase/HBaseKeys.java | 2 +-
.../providers/hbase/HBaseProviderService.java | 53 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
.../minicluster/live/TestHBaseMaster.groovy | 36 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 45 +-
89 files changed, 2663 insertions(+), 3397 deletions(-)
----------------------------------------------------------------------
[33/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/client/SliderClient.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/733745ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/733745ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/733745ea
Branch: refs/heads/develop
Commit: 733745eaf1cf3734dd8f25e7ff0945c253c4777f
Parents: f058495 41ec741
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 30 17:48:02 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 30 17:48:02 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 57 ++
app-packages/accumulo/appConfig.json | 57 --
app-packages/accumulo/pom.xml | 10 +
app-packages/accumulo/resources-default.json | 39 ++
app-packages/accumulo/resources.json | 39 --
app-packages/accumulo/src/assembly/accumulo.xml | 4 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +-
app-packages/hbase-win/README.txt | 3 -
app-packages/hbase-win/appConfig-default.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/hbase/resources-default.json | 3 +
app-packages/storm-win/README.txt | 36 ++
app-packages/storm-win/appConfig-default.json | 41 ++
.../storm-win/configuration/storm-env.xml | 65 +++
.../storm-win/configuration/storm-site.xml | 580 +++++++++++++++++++
app-packages/storm-win/metainfo.xml | 149 +++++
.../storm-win/package/scripts/drpc_server.py | 55 ++
.../storm-win/package/scripts/nimbus.py | 55 ++
.../storm-win/package/scripts/params.py | 56 ++
.../storm-win/package/scripts/rest_api.py | 57 ++
.../storm-win/package/scripts/service.py | 56 ++
.../storm-win/package/scripts/status_params.py | 37 ++
app-packages/storm-win/package/scripts/storm.py | 53 ++
.../storm-win/package/scripts/supervisor.py | 61 ++
.../storm-win/package/scripts/ui_server.py | 55 ++
.../storm-win/package/scripts/yaml_config.py | 80 +++
.../storm-win/package/templates/config.yaml.j2 | 28 +
.../package/templates/storm_jaas.conf.j2 | 44 ++
app-packages/storm-win/pom.xml | 91 +++
app-packages/storm-win/resources-default.json | 30 +
app-packages/storm-win/src/assembly/storm.xml | 68 +++
app-packages/storm/appConfig-default.json | 2 +-
slider-agent/src/main/python/kazoo/client.py | 56 +-
.../src/main/python/kazoo/handlers/utils.py | 42 +-
.../main/python/kazoo/protocol/connection.py | 74 +--
.../src/main/python/kazoo/tests/test_client.py | 55 +-
.../main/python/kazoo/tests/test_connection.py | 89 ++-
slider-core/pom.xml | 6 +
.../org/apache/slider/client/SliderClient.java | 291 +++++++++-
.../common/SliderXMLConfKeysForTesting.java | 1 +
.../common/params/ActionDiagnosticArgs.java | 66 +++
.../apache/slider/common/params/Arguments.java | 7 +
.../apache/slider/common/params/ClientArgs.java | 11 +-
.../slider/common/params/SliderActions.java | 4 +
.../apache/slider/common/tools/SliderUtils.java | 108 ++++
.../slider/providers/agent/AgentKeys.java | 5 +
.../providers/agent/AgentProviderService.java | 2 +-
.../services/security/CertificateManager.java | 48 +-
.../providers/slideram/instance/appconf.json | 3 +-
.../agent/TestAgentAMManagementWS.groovy | 131 +++--
.../apache/slider/test/SliderTestUtils.groovy | 3 +-
.../framework/AgentCommandTestBase.groovy | 7 +
.../funtest/lifecycle/AMFailuresIT.groovy | 6 +
.../clusters/remote/slider/slider-client.xml | 6 +
54 files changed, 2595 insertions(+), 344 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 99896c8,06c37ba..5d994f1
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -133,8 -136,8 +142,9 @@@ import java.io.IOException
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
+ import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@@ -2408,55 -2377,297 +2428,318 @@@ public class SliderClient extends Abstr
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public List<ServiceInstanceData> actionRegistryList(
+ public Collection<ServiceRecord> actionRegistryListYarn(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- SliderRegistryService registryService = getRegistry();
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
- List<CuratorServiceInstance<ServiceInstanceData>> instances =
- registryService.findInstances(serviceType, name);
- int size = instances.size();
- if (size == 0) {
- throw new FileNotFoundException("No entries for servicetype "
- + serviceType
- + " name " + name);
+ RegistryOperations operations = getRegistryOperations();
+ Collection<ServiceRecord> serviceRecords;
+ if (StringUtils.isEmpty(name)) {
+ String serviceclassPath =
+ serviceclassPath(
+ currentUser(),
+ serviceType);
+
+ try {
+ Map<String, ServiceRecord> recordMap =
+ listServiceRecords(operations, serviceclassPath);
+ RegistryPathStatus[] listDir;
+ if (recordMap.isEmpty()) {
+ throw new UnknownApplicationInstanceException(
+ "No applications registered under " + serviceclassPath);
+ }
+ serviceRecords = recordMap.values();
+ } catch (PathNotFoundException e) {
+ throw new UnknownApplicationInstanceException(e.getPath().toString(),
+ e);
+ }
+ } else {
+ ServiceRecord instance = lookupServiceRecord(registryArgs);
+ serviceRecords = new ArrayList<ServiceRecord>(1);
+ serviceRecords.add(instance);
}
- List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
- for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
- ServiceInstanceData payload = instance.payload;
- logInstance(payload, registryArgs.verbose);
- sids.add(payload);
+
+ for (ServiceRecord serviceRecord : serviceRecords) {
+ logInstance(serviceRecord, registryArgs.verbose);
}
- return sids;
+ return serviceRecords;
}
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException
+ * YARN problems
+ * @throws IOException
+ * Network or other problems
+ */
+ private int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ try {
+ if (diagnosticArgs.client) {
+ actionDiagnosticClient();
+ } else if (SliderUtils.isSet(diagnosticArgs.application)) {
+ actionDiagnosticApplication(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.slider)) {
+ actionDiagnosticSlider(diagnosticArgs);
+ } else if (diagnosticArgs.yarn) {
+ actionDiagnosticYarn(diagnosticArgs);
+ } else if (diagnosticArgs.credentials) {
+ actionDiagnosticCredentials();
+ } else if (SliderUtils.isSet(diagnosticArgs.all)) {
+ actionDiagnosticAll(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.level)) {
+ actionDiagnosticIntelligent(diagnosticArgs);
+ } else {
+ // it's an unknown command
+ log.info(ActionDiagnosticArgs.USAGE);
+ return EXIT_USAGE;
+ }
+ } catch (Exception e) {
+ log.error(e.toString());
+ return EXIT_FALSE;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException, URISyntaxException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.level;
+
+ try {
+ SliderUtils.validateClientConfigFile();
+ log.info("Slider-client.xml is accessible");
+ } catch (IOException e) {
+ // we are catching exceptions here because those are indication of
+ // validation result, and we need to print them here
+ log.error("validation of slider-client.xml fails because: "
+ + e.toString());
+ return;
+ }
+ SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
+ // cluster not found exceptions will be thrown upstream
+ ClusterDescription clusterDescription = clusterOperations
+ .getClusterDescription();
+ log.info("Slider AppMaster is accessible");
+
+ if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
+ AggregateConf instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, that means slider uploaded the agent tarball for the user
+ //and we need to use where slider has put
+ if(imagePath == null){
- ApplicationReport appReport = YARNRegistryClient.findInstance(clusterName);
++ ApplicationReport appReport = findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, imagePath);
+ log.info("Slider agent tarball is properly installed");
+ } catch (IOException e) {
+ log.error("can not find or open agent tar ball: " + e.toString());
+ return;
+ }
+ String pkgTarballPath = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ log.info("Application tarball is properly installed");
+ } catch (IOException e) {
+ log.error("can not find or open application tar ball: "
+ + e.toString());
+ return;
+ }
+ }
+ }
+
+ private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs)
+ throws IOException, YarnException {
+ //assign application name from param to each sub diagnostic function
+ diagnosticArgs.application = diagnosticArgs.all;
+ diagnosticArgs.slider = diagnosticArgs.all;
+ actionDiagnosticClient();
+ actionDiagnosticApplication(diagnosticArgs);
+ actionDiagnosticSlider(diagnosticArgs);
+ actionDiagnosticYarn(diagnosticArgs);
+ actionDiagnosticCredentials();
+ }
+
+ private void actionDiagnosticCredentials() throws BadConfigException, IOException
+ {
+ if (SliderUtils.isHadoopClusterSecure(SliderUtils
+ .loadClientConfigurationResource())) {
+ String credentialCacheFileDescription = null;
+ try {
+ credentialCacheFileDescription = SliderUtils
+ .checkCredentialCacheFile();
+ } catch (BadConfigException e) {
+ log.error("The credential config is not valid: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Unable to read the credential file: " + e.toString());
+ throw e;
+ }
+ log.info("Credential cache file for the current user: "
+ + credentialCacheFileDescription);
+ } else {
+ log.info("the cluster is not in secure mode");
+ }
+ }
+
+ private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException {
+ JSONObject converter = null;
+ log.info("the node in the YARN cluster has below state: ");
+ List<NodeReport> yarnClusterInfo;
+ try {
+ yarnClusterInfo = yarnClient.getNodeReports(NodeState.RUNNING);
+ } catch (YarnException e1) {
+ log.error("Exception happened when fetching node report from the YARN cluster: " + e1.toString());
+ throw e1;
+ } catch (IOException e1) {
+ log.error("Network problem happened when fetching node report YARN cluster: " + e1.toString());
+ throw e1;
+ }
+ for(NodeReport nodeReport : yarnClusterInfo){
+ log.info(nodeReport.toString());
+ }
+
+ if (diagnosticArgs.verbose) {
+ Writer configWriter = new StringWriter();
+ try {
+ Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter);
+ } catch (IOException e1) {
+ log.error("Network problem happened when retrieving YARN config from YARN: " + e1.toString());
+ throw e1;
+ }
+ try {
+ converter = new JSONObject(configWriter.toString());
+ log.info("the configuration of the YARN cluster is: "
+ + converter.toString(2));
+
+ } catch (JSONException e) {
+ log.error("JSONException happened during parsing response from YARN: " + e.toString());
+ }
+ }
+ }
+
+ private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.slider;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, it will be uploaded by Slider and thus at slider's path
+ if(imagePath == null){
- ApplicationReport appReport = YARNRegistryClient.findInstance(clusterName);
++ ApplicationReport appReport = findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ log.info("The path of slider agent tarball on HDFS is: " + imagePath);
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.application;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String clusterDir = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_ROOT);
+ String pkgTarball = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_DEF);
+ String runAsUser = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.RUNAS_USER);
+
+ log.info("The location of the cluster instance directory in HDFS is: "
+ + clusterDir);
+ log.info("The name of the application package tarball on HDFS is: "
+ + pkgTarball);
+ log.info("The runas user of the application in the cluster is: "
+ + runAsUser);
+
+ if (diagnosticArgs.verbose) {
+ log.info("App config of the application: "
+ + instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application: "
+ + instanceDefinition.getResources().toJson());
+ }
+ }
+
+ private void actionDiagnosticClient() throws SliderException, IOException {
+ String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+ String clientConfigPath = SliderUtils.getClientConfigPath();
+ String jdkInfo = SliderUtils.getJDKInfo();
+ log.info("The slider command path: " + currentCommandPath);
+ log.info("The slider-client.xml used by current running command path: "
+ + clientConfigPath);
+ log.info(jdkInfo);
+
+ try {
+ SliderUtils.validateSliderClientEnvironment(log);
+ } catch (SliderException e) {
+ log.error(e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error(e.toString());
+ throw e;
+ }
+ }
+
- private void logInstance(ServiceInstanceData instance,
++
+ /**
+ * Log a service record instance
+ * @param instance record
+ * @param verbose verbose logging of all external endpoints
+ */
+ private void logInstance(ServiceRecord instance,
boolean verbose) {
if (!verbose) {
- log.info("{}", instance.id);
+ log.info("{}", instance.yarn_id);
} else {
- log.info("{}: ", instance.id);
+ log.info("{}: ", instance.yarn_id);
logEndpoints(instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/733745ea/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------