You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/04 15:03:47 UTC
[01/11] git commit: SLIDER-149: service registry is on local hadoop
trunk builds only
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry [created] 7e1c77f89
SLIDER-149: service registry is on local hadoop trunk builds only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cfc6e1d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cfc6e1d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cfc6e1d0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: cfc6e1d062d7478aa9b48b3b1c88a611a5c0ab44
Parents: 7823d3a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jul 3 09:09:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jul 3 09:09:26 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cfc6e1d0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0076561..2f84d29 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,7 +115,7 @@
<!--
core artifacts
-->
- <hadoop.version>2.4.0</hadoop.version>
+ <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.3-hadoop2</hbase.version>
<accumulo.version>1.6.0</accumulo.version>
[08/11] git commit: SLIDER-141 registry coming up on server;
some test with the existing registry is now failing. Clash of roots?
Posted by st...@apache.org.
SLIDER-141 registry coming up on server; some test with the existing registry is now failing. Clash of roots?
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7607a61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7607a61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7607a61
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e7607a6179902c44c7dda2d66d7a6588e621ef71
Parents: bfbe312
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 1 21:56:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 1 21:56:30 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 87 +++++++++++++++-----
.../org/apache/slider/common/params/ArgOps.java | 2 +-
.../core/registry/YARNRegistryClient.java | 85 -------------------
.../slider/core/registry/YarnAppListClient.java | 86 +++++++++++++++++++
.../providers/AbstractProviderService.java | 2 +-
.../slider/providers/ProviderService.java | 3 +-
.../server/appmaster/SliderAppMaster.java | 19 +++--
.../server/appmaster/web/SliderAMWebApp.java | 7 +-
.../utility/AbstractSliderLaunchedService.java | 20 +++--
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../standalone/TestBuildStandaloneAM.groovy | 4 +-
.../standalone/TestStandaloneAgentAM.groovy | 11 ++-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 2 +-
.../publisher/TestAgentProviderService.java | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 2 +-
16 files changed, 200 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..6b67342 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.OptionKeys;
@@ -86,7 +88,7 @@ import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
-import org.apache.slider.core.registry.YARNRegistryClient;
+import org.apache.slider.core.registry.YarnAppListClient;
import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
@@ -154,10 +156,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Yarn client service
*/
private SliderYarnClientImpl yarnClient;
- private YARNRegistryClient YARNRegistryClient;
+ private YarnAppListClient YarnAppListClient;
private AggregateConf launchedInstanceDefinition;
private SliderRegistryService registry;
+
+ /**
+ * The YARN registry service
+ */
+ private YarnRegistryService yarnRegistry;
+
/**
* Constructor
*/
@@ -196,8 +204,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
//here the superclass is inited; getConfig returns a non-null value
sliderFileSystem = new SliderFileSystem(getConfig());
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
+ YarnAppListClient =
+ new YarnAppListClient(yarnClient, getUsername(), getConfig());
}
/**
@@ -1014,8 +1022,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
DEFAULT_REGISTRY_PATH);
+ addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.REGISTRY_ZK_QUORUM);
addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
-
+ define(commandLine, RegistryConstants.REGISTRY_ZK_QUORUM,
+ getYarnRegistry().getCurrentZookeeperQuorum());
+
+
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
@@ -1134,18 +1146,30 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String key) {
String val = conf.get(key);
if (val != null) {
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ define(cmdLine, key, val);
return true;
} else {
return false;
}
}
-
- private void addConfOptionToCLI(CommandLineBuilder cmdLine,
+
+ private String addConfOptionToCLI(CommandLineBuilder cmdLine,
Configuration conf,
- String key, String defVal) {
+ String key,
+ String defVal) {
String val = conf.get(key, defVal);
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
+ define(cmdLine, key, val);
+ return val;
+ }
+
+ /**
+ * Add a define command to the CLI
+ * @param cmdLine
+ * @param key
+ * @param val
+ */
+ private void define(CommandLineBuilder cmdLine, String key, String val) {
+ cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
}
private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
@@ -1331,7 +1355,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public List<ApplicationReport> listSliderInstances(String user)
throws YarnException, IOException {
- return YARNRegistryClient.listInstances();
+ return YarnAppListClient.listInstances();
}
/**
@@ -1498,8 +1522,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Get at the service registry operations
* @return registry client -valid after the service is inited.
*/
- public YARNRegistryClient getYARNRegistryClient() {
- return YARNRegistryClient;
+ public YarnAppListClient getYarnAppListClient() {
+ return YarnAppListClient;
}
/**
@@ -1512,7 +1536,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
- return YARNRegistryClient.findInstance(appname);
+ return YarnAppListClient.findInstance(appname);
}
private RunningApplication findApplication(String appname) throws
@@ -1532,7 +1556,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private List<ApplicationReport> findAllLiveInstances(String appname)
throws YarnException, IOException {
- return YARNRegistryClient.findAllLiveInstances(appname);
+ return YarnAppListClient.findAllLiveInstances(appname);
}
@@ -2307,8 +2331,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
public List<CuratorServiceInstance<ServiceInstanceData>> listRegistryInstances()
throws IOException, YarnException {
- maybeStartRegistry();
- return registry.listInstances(SliderKeys.APP_TYPE);
+ return getRegistry().listInstances(SliderKeys.APP_TYPE);
}
/**
@@ -2321,8 +2344,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
IOException,
YarnException {
try {
- maybeStartRegistry();
- return registry.instanceIDs(SliderKeys.APP_TYPE);
+ return getRegistry().instanceIDs(SliderKeys.APP_TYPE);
} catch (YarnException | IOException e) {
throw e;
} catch (Exception e) {
@@ -2361,6 +2383,33 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return maybeStartRegistry();
}
+
+ /**
+ * Start the registry if it is not there yet
+ * @return the registry service
+ * @throws SliderException
+ * @throws IOException
+ */
+ private synchronized YarnRegistryService maybeStartYarnRegistry() throws
+ SliderException,
+ IOException {
+
+ if (yarnRegistry == null) {
+ yarnRegistry = startYarnRegistryService();
+ }
+ return yarnRegistry;
+ }
+
+ /**
+ * Get the YARN registry
+ * @return the registry
+ */
+ public YarnRegistryService getYarnRegistry() throws
+ SliderException,
+ IOException {
+ return maybeStartYarnRegistry();
+ }
+
/**
* Output to standard out/stderr (implementation specific detail)
* @param src source
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index 0837dd2..f42e548 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -82,7 +82,7 @@ public class ArgOps {
for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
- log.debug("configuration[{}]=\"{}\"", key, val);
+ log.debug("configuration[{}]<=\"{}\"", key, val);
conf.set(key, val, "command line");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
deleted file mode 100644
index 31147ef..0000000
--- a/slider-core/src/main/java/org/apache/slider/core/registry/YARNRegistryClient.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.client.SliderYarnClientImpl;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Client code for interacting with a registry of service instances.
- * The initial logic just enumerates service instances in the YARN RM
- */
-public class YARNRegistryClient {
-
- final SliderYarnClientImpl yarnClient;
- final String username;
- final Configuration conf;
-
-
- public YARNRegistryClient(SliderYarnClientImpl yarnClient,
- String username,
- Configuration conf) {
- this.yarnClient = yarnClient;
- this.username = username;
- this.conf = conf;
- }
-
- /**
- * find all live instances of a specific app -if there is >1 in the cluster,
- * this returns them all. State should be running or less
- * @param appname application name
- * @return the list of all matching application instances
- */
- public List<ApplicationReport> findAllLiveInstances(String appname)
- throws YarnException, IOException {
-
- return yarnClient.findAllLiveInstances(username, appname);
- }
-
-
- /**
- * Find an instance of a application belong to the current user
- * @param appname application name
- * @return the app report or null if none is found
- * @throws YarnException YARN issues
- * @throws IOException IO problems
- */
- public ApplicationReport findInstance(String appname) throws
- YarnException,
- IOException {
- List<ApplicationReport> instances = listInstances();
- return yarnClient.findClusterInInstanceList(instances, appname);
- }
-
- /**
- * List instances belonging to a specific user
- * @return a possibly empty list of AMs
- */
- public List<ApplicationReport> listInstances()
- throws YarnException, IOException {
- return yarnClient.listInstances(username);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
new file mode 100644
index 0000000..0ca0518
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Client code for interacting with a list of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YarnAppListClient {
+
+ final SliderYarnClientImpl yarnClient;
+ final String username;
+ final Configuration conf;
+
+
+ public YarnAppListClient(SliderYarnClientImpl yarnClient,
+ String username,
+ Configuration conf) {
+ this.yarnClient = yarnClient;
+ this.username = username;
+ this.conf = conf;
+ }
+
+ /**
+ * find all live instances of a specific app -if there is more than one
+ * in the cluster, this returns them all. State should be running or earlier
+ * in the lifecycle
+ * @param appname application name
+ * @return the list of all matching application instances
+ */
+ public List<ApplicationReport> findAllLiveInstances(String appname)
+ throws YarnException, IOException {
+
+ return yarnClient.findAllLiveInstances(username, appname);
+ }
+
+
+ /**
+ * Find an instance of a application belong to the current user
+ * @param appname application name
+ * @return the app report or null if none is found
+ * @throws YarnException YARN issues
+ * @throws IOException IO problems
+ */
+ public ApplicationReport findInstance(String appname) throws
+ YarnException,
+ IOException {
+ List<ApplicationReport> instances = listInstances();
+ return yarnClient.findClusterInInstanceList(instances, appname);
+ }
+
+ /**
+ * List instances belonging to a specific user
+ * @return a possibly empty list of AMs
+ */
+ public List<ApplicationReport> listInstances()
+ throws YarnException, IOException {
+ return yarnClient.listInstances(username);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index b60328f..b50b09d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,7 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index ad0e25d..6e2bc92 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -40,7 +40,6 @@ import org.apache.slider.server.services.registry.RegistryViewForProviders;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e469d6b..87c5295 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.hadoop.yarn.registry.client.types.RegistryTypeUtils;
import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -290,7 +290,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* The registry service
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private SliderRegistryService registry;
+ private SliderRegistryService sliderRegistry;
/**
@@ -601,8 +601,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//registry
- registry = startRegistrationService();
+ log.info("Starting slider registry");
+ sliderRegistry = startRegistrationService();
+ log.info("Starting Yarn registry");
yarnRegistry = startYarnRegistryService();
+ log.info(yarnRegistry.toString());
//build the role map
List<ProviderRole> providerRoles =
@@ -617,7 +620,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
startAgentWebApp(appInformation, serviceConf);
- webApp = new SliderAMWebApp(registry);
+ webApp = new SliderAMWebApp(sliderRegistry, yarnRegistry);
WebApps.$for(SliderAMWebApp.BASE_PATH, WebAppApi.class,
new WebAppApiImpl(this,
stateForProviders,
@@ -727,8 +730,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, yarnRegistry, this);
- sliderAMProvider.bind(stateForProviders, registry, yarnRegistry, null);
+ providerService.bind(stateForProviders, sliderRegistry, yarnRegistry, this);
+ sliderAMProvider.bind(stateForProviders, sliderRegistry, yarnRegistry, null);
// now do the registration
registerServiceInstance(clustername, appid);
@@ -806,7 +809,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
RegistryNaming.createRegistryName(instanceName, service_user_name,
serviceName, id);
- List<String> serviceInstancesRunning = registry.instanceIDs(serviceName);
+ List<String> serviceInstancesRunning = sliderRegistry.instanceIDs(serviceName);
log.info("service instances already running: {}", serviceInstancesRunning);
@@ -847,7 +850,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// push the registration info to ZK
- registry.registerSelf(instanceData, unsecureWebAPI);
+ sliderRegistry.registerSelf(instanceData, unsecureWebAPI);
yarnRegistry.putServiceEntry(service_user_name,
SliderKeys.APP_TYPE, instanceName,
registryEntry );
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 4f290af..03d4f28 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -22,6 +22,7 @@ import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.spi.container.servlet.ServletContainer;
import org.apache.curator.x.discovery.ServiceDiscovery;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.hadoop.yarn.webapp.Dispatcher;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
@@ -48,8 +49,12 @@ public class SliderAMWebApp extends WebApp {
public static final String CLUSTER_SPEC = "/spec";
public final SliderRegistryService registry;
+ private final YarnRegistryService yarnRegistry;
- public SliderAMWebApp(SliderRegistryService registry) {
+
+ public SliderAMWebApp(SliderRegistryService registry,
+ YarnRegistryService yarnRegistry) {
+ this.yarnRegistry = yarnRegistry;
Preconditions.checkArgument(registry != null, "registry null");
this.registry = registry;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index e7c678d..d059f34 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -22,7 +22,7 @@ package org.apache.slider.server.services.utility;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
@@ -49,6 +49,13 @@ public abstract class AbstractSliderLaunchedService extends
new YarnConfiguration();
}
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ String quorum = lookupZKQuorum();
+ conf.set(RegistryConstants.REGISTRY_ZK_QUORUM, quorum);
+ super.serviceInit(conf);
+ }
+
/**
* Start the registration service
* @return the instance
@@ -70,15 +77,14 @@ public abstract class AbstractSliderLaunchedService extends
* @throws BadConfigException if it is not there or invalid
*/
public String lookupZKQuorum() throws BadConfigException {
- String registryQuorum = getConfig().get(
- RegistryConstants.ZK_QUORUM);
+ String registryQuorum = getConfig().get(RegistryConstants.REGISTRY_ZK_QUORUM);
registryQuorum = getConfig().get(
SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
registryQuorum);
if (SliderUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the"
- + " configuration property " + RegistryConstants.ZK_QUORUM
+ + " configuration property " + RegistryConstants.REGISTRY_ZK_QUORUM
);
}
ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
@@ -113,12 +119,10 @@ public abstract class AbstractSliderLaunchedService extends
public YarnRegistryService startYarnRegistryService()
throws BadConfigException {
- String quorum = lookupZKQuorum();
Configuration conf = getConfig();
// push back the slider registry entry if needed
- if (conf.get(RegistryConstants.ZK_QUORUM) == null) {
- conf.set(RegistryConstants.ZK_QUORUM, quorum);
- }
+ String quorum = lookupZKQuorum();
+ conf.set(RegistryConstants.REGISTRY_ZK_QUORUM, quorum);
YarnRegistryService yarnRegistryService =
new YarnRegistryService("YarnRegistry");
deployChildService(yarnRegistryService);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 7d62f37..b496193 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -107,7 +107,7 @@ class TestActionList extends AgentMiniClusterTestBase {
//now look for the explicit sevice
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
assert instance != null
log.info(instance.toString())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 653a751..711bc24 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -65,7 +65,7 @@ class TestBuildStandaloneAM extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
@@ -115,7 +115,7 @@ class TestBuildStandaloneAM extends AgentMiniClusterTestBase {
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 3d9b9cd..0335b6e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -80,7 +80,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
String username = client.username
- def serviceRegistryClient = client.YARNRegistryClient
+ def serviceRegistryClient = client.yarnAppListClient
describe("list of all applications")
logApplications(apps)
describe("apps of user $username")
@@ -92,7 +92,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
logReport(instance)
assert instance != null
- //switch to the ZK-based registry
+ //switch to the slider ZK-based registry
describe "service registry names"
SliderRegistryService registry = client.registry
@@ -106,13 +106,16 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
instanceIds.each { String it -> log.info(it) }
describe "service registry slider instances"
- List<CuratorServiceInstance<ServiceInstanceData>> instances = client.listRegistryInstances(
- )
+ List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ client.listRegistryInstances()
instances.each { CuratorServiceInstance<ServiceInstanceData> svc ->
log.info svc.toString()
}
describe "end list service registry slider instances"
+ describe "Yarn registry"
+ def yarnRegistry = client.yarnRegistry
+
describe "teardown of cluster instance #1"
//now kill that cluster
assert 0 == clusterActionFreeze(client, clustername)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index de9f184..4806adb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -97,7 +97,7 @@ class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
String username = client.username
- def yarnRegistryClient = client.YARNRegistryClient
+ def yarnRegistryClient = client.yarnAppListClient
describe("list of all applications")
logApplications(apps)
describe("apps of user $username")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 43bd8f4..3730f5b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.service.LifecycleEvent
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService
import org.apache.hadoop.yarn.registry.client.types.ServiceEntry
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 3f2c7fd..e35deb8 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -16,7 +16,7 @@
*/
package org.apache.slider.server.appmaster.web.rest.publisher;
-import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7607a61/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
index c305b5b..c22ad88 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
@@ -57,7 +57,7 @@ class TestBuildThawClusterM1W1 extends HBaseMiniClusterTestBase {
[:])
SliderClient sliderClient = launcher.service
addToTeardown(sliderClient);
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
[03/11] git commit: SLIDER-149 javadocs and whitespace
Posted by st...@apache.org.
SLIDER-149 javadocs and whitespace
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9ec12883
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9ec12883
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9ec12883
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 9ec1288318776332cee9544411a50706700f305c
Parents: edd96e6
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jul 9 12:02:08 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jul 9 12:02:08 2014 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/core/persist/JsonSerDeser.java | 2 +-
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ec12883/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 3512168..792dcdf 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -197,7 +197,7 @@ public class JsonSerDeser<T> {
/**
* Convert an object to a JSON string
- * @param o object to convert
+ * @param instance instance to convert
* @return a JSON string description
* @throws JsonParseException parse problems
* @throws JsonMappingException O/J mapping problems
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ec12883/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 2c1b270..47a67ac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -129,6 +129,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
KEY_TEST_TIMEOUT,
DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
)
+
@BeforeClass
public static void checkWindowsSupport() {
if (Shell.WINDOWS) {
[11/11] git commit: SLIDER-149 YARN and slider registries living side
by side, though YARN reg is doing simple registration only
Posted by st...@apache.org.
SLIDER-149 YARN and slider registries living side by side, though YARN reg is doing simple registration only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e1c77f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e1c77f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e1c77f8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7e1c77f891fa797f12b282417c1bb8b21d6c4e18
Parents: 96f4acb
Author: Steve Loughran <st...@apache.org>
Authored: Mon Aug 4 14:03:02 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Aug 4 14:03:02 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 26 ++++++++++++++++++--
.../server/appmaster/SliderAppMaster.java | 2 ++
.../server/services/curator/CuratorHelper.java | 5 ++++
.../server/services/curator/CuratorService.java | 17 +++++++++++++
.../utility/AbstractSliderLaunchedService.java | 6 ++---
.../standalone/TestStandaloneRegistryAM.groovy | 5 ++++
6 files changed, 56 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 6b67342..87af625 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper;
import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -666,7 +667,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
@@ -1233,7 +1234,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
private boolean getUsingMiniMRCluster() {
return getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
- false);
+ false);
}
/**
@@ -2334,6 +2335,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return getRegistry().listInstances(SliderKeys.APP_TYPE);
}
+
+ /**
+ * Get an on-demand path jumper
+ * @return a class that can dump the contents of the registry
+ */
+ @VisibleForTesting
+ public ZKPathDumper dumpSliderRegistry() throws SliderException, IOException {
+ return getRegistry().dumpPath();
+ }
+
/**
* List instances in the registry
* @return the instance IDs
@@ -2410,6 +2421,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return maybeStartYarnRegistry();
}
+
+ /**
+ * Get an on-demand path jumper
+ * @return a class that can dump the contents of the registry
+ */
+ @VisibleForTesting
+ public ZKPathDumper dumpYarnRegistry() throws SliderException, IOException {
+ return getYarnRegistry().dumpPath();
+ }
+
+
/**
* Output to standard out/stderr (implementation specific detail)
* @param src source
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 87c5295..236c5b6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -603,6 +603,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//registry
log.info("Starting slider registry");
sliderRegistry = startRegistrationService();
+ log.info(sliderRegistry.toString());
+
log.info("Starting Yarn registry");
yarnRegistry = startYarnRegistryService();
log.info(yarnRegistry.toString());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
index a6316d5..72d413f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorHelper.java
@@ -135,4 +135,9 @@ public class CuratorHelper extends Configured {
ServiceInstanceData.class);
}
+
+ @Override
+ public String toString() {
+ return "Curator binding " + connectionString;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
index b5b0c14..cfdf80d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
@@ -18,12 +18,14 @@
package org.apache.slider.server.services.curator;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -95,4 +97,19 @@ public class CuratorService extends AbstractService {
public final CuratorHelper getCuratorHelper() {
return curatorHelper;
}
+
+ @Override
+ public String toString() {
+ return super.toString() + "; "
+ + (curatorHelper != null ? curatorHelper : "( unbound)")
+ + "; " + basePath;
+ }
+
+ /**
+ * Get an on-demand path jumper
+ * @return a class that can dump the contents of the registry
+ */
+ public ZKPathDumper dumpPath() {
+ return new ZKPathDumper(curator, basePath);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index d059f34..4b78d52 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -65,9 +65,9 @@ public abstract class AbstractSliderLaunchedService extends
throws BadConfigException {
String registryQuorum = lookupZKQuorum();
- String zkPath = getConfig().get(RegistryConstants.REGISTRY_ZK_ROOT,
- RegistryConstants.DEFAULT_REGISTRY_ROOT);
- zkPath = getConfig().get(REGISTRY_PATH, zkPath);
+ String zkPath = getConfig().get(
+ SliderXmlConfKeys.REGISTRY_PATH,
+ SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
return startSliderRegistrationService(registryQuorum, zkPath);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e1c77f8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index 4806adb..7e8ea02 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -111,6 +111,11 @@ class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
//switch to the ZK-based registry
+
+ log.info("slider service registry: \n${client.dumpSliderRegistry()}\n")
+ log.info("yarn service registry: \n${client.dumpYarnRegistry()}\n")
+
+
describe "service registry names"
SliderRegistryService registryService = client.registry
def serviceTypes = registryService.serviceTypes;
[07/11] git commit: Merge remote-tracking branch 'apache/develop'
into feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge remote-tracking branch 'apache/develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfbe3125
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfbe3125
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfbe3125
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bfbe31253c0fd90d956adc14fef024219311ecf2
Parents: d8d3671 8c6c6b6
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 1 20:46:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 1 20:46:30 2014 +0100
----------------------------------------------------------------------
slider-assembly/pom.xml | 6 +--
.../org/apache/slider/common/SliderKeys.java | 2 +-
.../agent/actions/TestActionExists.groovy | 2 +-
.../slider/agent/actions/TestActionList.groovy | 2 +-
.../agent/actions/TestActionStatus.groovy | 6 +--
.../agent/freezethaw/TestFreezeCommands.groovy | 3 +-
.../TestFreezeThawMasterlessAM.groovy | 3 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 3 +-
.../freezethaw/TestThawUnknownCluster.groovy | 3 +-
.../agent/standalone/TestBadAMHeap.groovy | 3 +-
.../agent/standalone/TestBadYarnQueue.groovy | 3 +-
.../standalone/TestBuildStandaloneAM.groovy | 6 +--
.../standalone/TestDestroyMasterlessAM.groovy | 3 +-
.../standalone/TestKillMasterlessAM.groovy | 3 +-
.../standalone/TestStandaloneAgentAM.groovy | 3 +-
.../standalone/TestStandaloneRegistryAM.groovy | 3 +-
.../agent/TestAgentAMManagementWS.groovy | 4 +-
.../slider/providers/agent/TestAgentEcho.groovy | 4 +-
.../providers/agent/TestBuildBasicAgent.groovy | 39 ++++++++------------
.../curator/TestRegistryRestResources.groovy | 10 +++--
.../publisher/TestPublisherRestResources.groovy | 5 +--
.../apache/slider/test/SliderTestBase.groovy | 17 ++++++++-
.../slider/test/YarnMiniClusterTestBase.groovy | 21 +++++------
.../test/YarnZKMiniClusterTestBase.groovy | 22 +++++++++--
.../providers/accumulo/AccumuloTestBase.groovy | 2 +-
.../live/TestAccCorrectInstanceName.groovy | 4 +-
.../live/TestAccFlexTablets133Mgr113.groovy | 3 +-
.../accumulo/live/TestAccFlexTablets1to3.groovy | 2 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 4 +-
.../live/TestAccLiveLocalArchive.groovy | 4 +-
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 3 +-
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 4 +-
.../accumulo/live/TestAccumuloAMWebApp.groovy | 4 +-
.../live/TestInvalidMonitorAddress.groovy | 3 +-
.../TestFreezeThawClusterFromArchive.groovy | 3 +-
.../archives/TestLiveClusterFromArchive.groovy | 4 +-
.../TestLiveClusterFromArchiveOnHDFS.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
.../failures/TestFailedRegionService.groovy | 7 ++--
.../failures/TestFailureThreshold.groovy | 5 +--
.../failures/TestKilledHBaseAM.groovy | 3 +-
.../failures/TestKilledHBaseMaster.groovy | 4 +-
.../flexing/TestClusterFlex0To1.groovy | 2 +-
.../flexing/TestClusterFlex1To1.groovy | 4 +-
.../flexing/TestClusterFlex1To2.groovy | 4 +-
.../flexing/TestClusterFlex2DownTo1.groovy | 2 +-
.../flexing/TestClusterFlex2To5.groovy | 4 +-
.../flexing/TestClusterFlexDownToZero.groovy | 4 +-
.../flexing/TestHMasterFlex1To2.groovy | 4 +-
...reezeReconfigureThawLiveRegionService.groovy | 3 +-
.../TestFreezeThawLiveRegionService.groovy | 3 +-
.../minicluster/live/Test2Master2RS.groovy | 3 +-
.../minicluster/live/TestHBaseMaster.groovy | 3 +-
.../live/TestHBaseMasterOnHDFS.groovy | 4 +-
.../live/TestHBaseMasterWithBadHeap.groovy | 3 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 4 +-
.../live/TestLiveTwoNodeRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 3 +-
.../masterless/TestRoleOptPropagation.groovy | 6 +--
.../TestSliderConfDirToMasterlessAM.groovy | 3 +-
61 files changed, 146 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfbe3125/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
[09/11] git commit: SLIDER-149 mark some CuratorService methods as
final
Posted by st...@apache.org.
SLIDER-149 mark some CuratorService methods as final
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c2dadcb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c2dadcb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c2dadcb7
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c2dadcb7051b52089ebc927ff4e30befeadb44e3
Parents: e7607a6
Author: Steve Loughran <st...@apache.org>
Authored: Sat Aug 2 13:49:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Aug 2 13:49:36 2014 +0100
----------------------------------------------------------------------
.../slider/server/services/curator/CuratorService.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c2dadcb7/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
index 645bc8f..b5b0c14 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/CuratorService.java
@@ -65,11 +65,11 @@ public class CuratorService extends AbstractService {
closeCuratorComponent(curator);
}
- public CuratorFramework getCurator() {
+ public final CuratorFramework getCurator() {
return curator;
}
- protected void closeCuratorComponent(Closeable closeable) {
+ protected final void closeCuratorComponent(Closeable closeable) {
try {
IOUtils.closeStream(closeable);
} catch (Throwable ignored) {
@@ -78,21 +78,21 @@ public class CuratorService extends AbstractService {
}
}
- public String pathForServicetype(String servicetype) {
+ public final String pathForServicetype(String servicetype) {
return ZKPaths.makePath(getBasePath(), servicetype);
}
- protected String pathForInstance(String servicetype, String id) {
+ protected final String pathForInstance(String servicetype, String id) {
Preconditions.checkNotNull(servicetype);
Preconditions.checkNotNull(id);
return ZKPaths.makePath(pathForServicetype(servicetype), id);
}
- public String getBasePath() {
+ public final String getBasePath() {
return basePath;
}
- public CuratorHelper getCuratorHelper() {
+ public final CuratorHelper getCuratorHelper() {
return curatorHelper;
}
}
[10/11] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96f4acb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96f4acb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96f4acb0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 96f4acb082eefa7affd22878f4a603e3b6702280
Parents: c2dadcb 067c306
Author: Steve Loughran <st...@apache.org>
Authored: Sat Aug 2 13:49:40 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Aug 2 13:49:40 2014 +0100
----------------------------------------------------------------------
slider-agent/src/test/python/unitTests.py | 3 ++-
.../java/org/apache/slider/core/launch/AbstractLauncher.java | 3 +++
.../org/apache/slider/core/launch/JavaCommandLineBuilder.java | 3 +++
.../hbase/minicluster/flexing/TestClusterFlexDownToZero.groovy | 2 +-
.../providers/hbase/minicluster/live/TestTwoLiveClusters.groovy | 4 ++--
5 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[06/11] git commit: SLIDER-149 yarn-registry alongside slider registry
Posted by st...@apache.org.
SLIDER-149 yarn-registry alongside slider registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d8d3671d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d8d3671d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d8d3671d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d8d3671da22b65b67ee94ff620e39b6661722130
Parents: adc4640
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 1 19:55:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 1 19:55:15 2014 +0100
----------------------------------------------------------------------
.../core/registry/info/RegisteredEndpoint.java | 41 +++-----------
.../providers/AbstractProviderService.java | 24 ++++----
.../slider/providers/ProviderService.java | 14 +++--
.../providers/agent/AgentProviderService.java | 25 +++++++--
.../slideram/SliderAMProviderService.java | 9 +--
.../server/appmaster/SliderAppMaster.java | 58 ++++++++++++++------
.../utility/AbstractSliderLaunchedService.java | 53 +++++++++++++++---
.../model/mock/MockProviderService.groovy | 8 ++-
.../publisher/TestAgentProviderService.java | 7 ++-
.../providers/hbase/HBaseProviderService.java | 8 ++-
10 files changed, 155 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
index f3477d0..8c065e2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegisteredEndpoint.java
@@ -18,6 +18,8 @@
package org.apache.slider.core.registry.info;
+import org.apache.hadoop.yarn.registry.client.types.AddressTypes;
+import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -37,58 +39,29 @@ public class RegisteredEndpoint {
/**
* URL: {@value}
*/
- public static final String TYPE_URL = "url";
+ public static final String TYPE_URL = AddressTypes.ADDRESS_URI;
- /**
- * hostname: {@value}
- */
- public static final String TYPE_HOSTNAME = "hostname";
/**
* "hostname:port" pair: {@value}
*/
- public static final String TYPE_INETADDRESS = "inetaddress";
+ public static final String TYPE_INETADDRESS = AddressTypes.ADDRESS_HOSTNAME_AND_PORT;
/**
* simple path string: {@value}
*/
- public static final String TYPE_PATH = "path";
+ public static final String TYPE_PATH = AddressTypes.ADDRESS_PATH;
// standard protocols
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_TCP = "tcp";
-
- /**
- * Generic TCP protocol: {@value}
- */
-
- public static final String PROTOCOL_UDP = "udp";
-
- /**
- * HTTP: {@value}
- */
-
- public static final String PROTOCOL_HTTP = "http";
-
- /**
- * HTTPS: {@value}
- */
-
- public static final String PROTOCOL_HTTPS = "http";
-
/**
* Classic "Writable" Hadoop IPC: {@value}
*/
- public static final String PROTOCOL_HADOOP_RPC = "org.apache.hadoop.ipc.RPC";
+ public static final String PROTOCOL_HADOOP_RPC = ProtocolTypes.PROTOCOL_HADOOP_IPC;
/**
* Protocol buffer based Hadoop IPC: {@value}
*/
- public static final String PROTOCOL_HADOOP_PROTOBUF = "org.apache.hadoop.ipc.Protobuf";
+ public static final String PROTOCOL_HADOOP_PROTOBUF = ProtocolTypes.PROTOCOL_HADOOP_IPC_PROTOBUF;
/**
* The address -format is driven by the type entry
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e35227c..b60328f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,8 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -42,7 +44,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -69,6 +70,7 @@ public abstract class AbstractProviderService
protected ServiceInstanceData registryInstanceData;
protected AMViewForProviders amView;
protected URL amWebAPI;
+ private YarnRegistryService yarnRegistry;
public AbstractProviderService(String name) {
super(name);
@@ -93,9 +95,12 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
+ RegistryViewForProviders reg,
+ YarnRegistryService yarnRegistry,
+ AMViewForProviders amView) {
this.amState = stateAccessor;
this.registry = reg;
+ this.yarnRegistry = yarnRegistry;
this.amView = amView;
}
@@ -297,11 +302,6 @@ public abstract class AbstractProviderService
@Override
public void buildEndpointDetails(Map<String, String> details) {
ServiceInstanceData self = registry.getSelfRegistration();
- buildEndpointDetails(details, self);
- }
-
- public static void buildEndpointDetails(Map<String, String> details,
- ServiceInstanceData self) {
Map<String, RegisteredEndpoint> endpoints =
self.getRegistryView(true).endpoints;
for (Map.Entry<String, RegisteredEndpoint> endpoint : endpoints.entrySet()) {
@@ -311,13 +311,15 @@ public abstract class AbstractProviderService
}
}
}
+
@Override
public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException {
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData,
+ ServiceEntry serviceEntry)
+ throws IOException {
- this.amWebAPI = unsecureWebAPI;
+ this.amWebAPI = unsecureWebAPI;
this.registryInstanceData = registryInstanceData;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 56e24e9..ad0e25d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -139,11 +141,12 @@ public interface ProviderService extends ProviderCore, Service,
* bind operation -invoked before the service is started
* @param stateAccessor interface offering read access to the state
* @param registry
+ * @param yarnRegistry
* @param amView
*/
void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
- AMViewForProviders amView);
+ RegistryViewForProviders registry,
+ YarnRegistryService yarnRegistry, AMViewForProviders amView);
/**
* Returns the agent rest operations interface.
@@ -162,9 +165,10 @@ public interface ProviderService extends ProviderCore, Service,
* @param unsecureWebAPI
* @param secureWebAPI
* @param registryInstanceData
+ * @param serviceEntry
*/
void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException;
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData,
+ ServiceEntry serviceEntry) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a73aaab..4d0254d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.registry.client.types.Endpoint;
+import org.apache.hadoop.yarn.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -1018,19 +1021,29 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
+ URL secureWebAPI,
+ ServiceInstanceData instanceData, ServiceEntry serviceEntry) throws IOException {
super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ secureWebAPI,
+ instanceData,
+ serviceEntry);
try {
+ URL url = new URL(secureWebAPI, SLIDER_PATH_AGENTS);
+
+ URI uri = url.toURI();
instanceData.internalView.endpoints.put(
CustomRegistryConstants.AGENT_REST_API,
new RegisteredEndpoint(
- new URL(secureWebAPI, SLIDER_PATH_AGENTS),
+ url,
"Agent REST API"));
+
+ serviceEntry.putInternalEndpoint(
+ CustomRegistryConstants.AGENT_REST_API,
+ new Endpoint(CustomRegistryConstants.AGENT_REST_API,
+ ProtocolTypes.PROTOCOL_REST,
+ "Agent REST API",
+ uri));
} catch (URISyntaxException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index 184c25a..5eb3cb4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -111,12 +112,12 @@ public class SliderAMProviderService extends AbstractProviderService implements
@Override
public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
+ URL secureWebAPI,
+ ServiceInstanceData instanceData, ServiceEntry serviceEntry) throws IOException {
super.applyInitialRegistryDefinitions(unsecureWebAPI,
secureWebAPI,
- instanceData
- );
+ instanceData,
+ serviceEntry);
// now publish site.xml files
YarnConfiguration defaultYarnConfig = new YarnConfiguration();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 3b061d5..e469d6b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -50,6 +50,9 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
+import org.apache.hadoop.yarn.registry.client.types.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -179,6 +182,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
+ public static final String SLIDER_AM_RPC = "Slider AM RPC";
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
@@ -288,6 +292,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private SliderRegistryService registry;
+
+ /**
+ * The YARN registry service
+ */
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
+ private YarnRegistryService yarnRegistry;
+
+
/**
* Record of the max no. of cores allowed in this cluster
*/
@@ -590,6 +602,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//registry
registry = startRegistrationService();
+ yarnRegistry = startYarnRegistryService();
//build the role map
List<ProviderRole> providerRoles =
@@ -714,8 +727,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, this);
- sliderAMProvider.bind(stateForProviders, registry, null);
+ providerService.bind(stateForProviders, registry, yarnRegistry, this);
+ sliderAMProvider.bind(stateForProviders, registry, yarnRegistry, null);
// now do the registration
registerServiceInstance(clustername, appid);
@@ -785,7 +798,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
URL secureWebAPI = new URL(agentAccessUrl);
String serviceName = SliderKeys.APP_TYPE;
int id = appid.getId();
- String appServiceType = RegistryNaming.createRegistryServiceType(
+ String serviceType = RegistryNaming.createRegistryServiceType(
instanceName,
service_user_name,
serviceName);
@@ -797,37 +810,48 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info("service instances already running: {}", serviceInstancesRunning);
+ // slider instance data
ServiceInstanceData instanceData = new ServiceInstanceData(registryId,
- appServiceType);
+ serviceType);
+ // Yarn registry
+ ServiceEntry registryEntry = new ServiceEntry();
// IPC services
instanceData.externalView.endpoints.put(
CustomRegistryConstants.AM_IPC_PROTOCOL,
new RegisteredEndpoint(rpcServiceAddress,
RegisteredEndpoint.PROTOCOL_HADOOP_PROTOBUF,
- "Slider AM RPC") );
-
-
+ SLIDER_AM_RPC) );
+
+ registryEntry.putExternalEndpoint(CustomRegistryConstants.AM_IPC_PROTOCOL,
+ RegistryTypeUtils.ipcEndpoint(
+ CustomRegistryConstants.AM_IPC_PROTOCOL,
+ SLIDER_AM_RPC,
+ true,
+ RegistryTypeUtils.toWireFormat(rpcServiceAddress)));
+
// internal services
-
+
sliderAMProvider.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ secureWebAPI,
+ instanceData,
+ registryEntry);
// provider service dynamic definitions.
providerService.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
+ secureWebAPI,
+ instanceData,
+ registryEntry);
// push the registration info to ZK
- registry.registerSelf(
- instanceData, unsecureWebAPI
- );
+ registry.registerSelf(instanceData, unsecureWebAPI);
+ yarnRegistry.putServiceEntry(service_user_name,
+ SliderKeys.APP_TYPE, instanceName,
+ registryEntry );
+
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 6c0edb8..e7c678d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -19,7 +19,10 @@
package org.apache.slider.server.services.utility;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
@@ -30,7 +33,6 @@ import org.apache.slider.server.services.registry.SliderRegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_REGISTRY_PATH;
import static org.apache.slider.common.SliderXmlConfKeys.REGISTRY_PATH;
/**
@@ -56,8 +58,10 @@ public abstract class AbstractSliderLaunchedService extends
throws BadConfigException {
String registryQuorum = lookupZKQuorum();
- String zkPath = getConfig().get(REGISTRY_PATH, DEFAULT_REGISTRY_PATH);
- return startRegistrationService(registryQuorum, zkPath);
+ String zkPath = getConfig().get(RegistryConstants.REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_REGISTRY_ROOT);
+ zkPath = getConfig().get(REGISTRY_PATH, zkPath);
+ return startSliderRegistrationService(registryQuorum, zkPath);
}
/**
@@ -67,11 +71,14 @@ public abstract class AbstractSliderLaunchedService extends
*/
public String lookupZKQuorum() throws BadConfigException {
String registryQuorum = getConfig().get(
- SliderXmlConfKeys.REGISTRY_ZK_QUORUM);
+ RegistryConstants.ZK_QUORUM);
+ registryQuorum = getConfig().get(
+ SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+ registryQuorum);
if (SliderUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the"
- + " configuration property " + SliderXmlConfKeys.REGISTRY_ZK_QUORUM
+ + " configuration property " + RegistryConstants.ZK_QUORUM
);
}
ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
@@ -84,8 +91,9 @@ public abstract class AbstractSliderLaunchedService extends
* @param zkPath
* @return
*/
- public SliderRegistryService startRegistrationService(
- String zkConnection, String zkPath) {
+ public SliderRegistryService startSliderRegistrationService(
+ String zkConnection,
+ String zkPath) {
CuratorHelper curatorHelper =
new CuratorHelper(getConfig(), zkConnection);
@@ -96,9 +104,36 @@ public abstract class AbstractSliderLaunchedService extends
return registryBinderService;
}
- protected void requireArgumentSet(String argname, String argfield)
+ /**
+ * Start the YARN registration service
+ * @param zkConnection
+ * @param zkPath
+ * @return
+ */
+ public YarnRegistryService startYarnRegistryService()
+ throws BadConfigException {
+
+ String quorum = lookupZKQuorum();
+ Configuration conf = getConfig();
+ // push back the slider registry entry if needed
+ if (conf.get(RegistryConstants.ZK_QUORUM) == null) {
+ conf.set(RegistryConstants.ZK_QUORUM, quorum);
+ }
+ YarnRegistryService yarnRegistryService =
+ new YarnRegistryService("YarnRegistry");
+ deployChildService(yarnRegistryService);
+ return yarnRegistryService;
+ }
+
+ /**
+ * Utility method to require an argument to be set (non null, non-empty)
+ * @param argname argument name
+ * @param value value
+ * @throws BadCommandArgumentsException if the condition is not met
+ */
+ protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
- if (isUnset(argfield)) {
+ if (isUnset(value)) {
throw new BadCommandArgumentsException("Required argument "
+ argname
+ " missing");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 7b73451..43bd8f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -23,6 +23,8 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.service.LifecycleEvent
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
@@ -195,6 +197,7 @@ class MockProviderService implements ProviderService {
void bind(
StateAccessForProviders stateAccessor,
RegistryViewForProviders registry,
+ YarnRegistryService yarnRegistry,
AMViewForProviders amView) {
}
@@ -225,7 +228,10 @@ class MockProviderService implements ProviderService {
@Override
void applyInitialRegistryDefinitions(
- URL unsecureWebAPI, URL secureWebAPI, ServiceInstanceData registryInstanceData)
+ URL unsecureWebAPI,
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData,
+ ServiceEntry serviceEntry)
throws MalformedURLException, IOException {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 2427009..3f2c7fd 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.server.appmaster.web.rest.publisher;
+import org.apache.hadoop.yarn.registry.client.binding.zk.YarnRegistryService;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -40,8 +41,10 @@ public class TestAgentProviderService extends AgentProviderService{
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
- super.bind(stateAccessor, reg, amView);
+ RegistryViewForProviders reg,
+ YarnRegistryService yarnRegistry,
+ AMViewForProviders amView) {
+ super.bind(stateAccessor, reg, yarnRegistry, amView);
Map<String,String> dummyProps = new HashMap<>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8d3671d/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 6fa7ca8..dccb805 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,6 +22,7 @@ package org.apache.slider.providers.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.OptionKeys;
@@ -210,10 +211,11 @@ public class HBaseProviderService extends AbstractProviderService implements
@Override
public void applyInitialRegistryDefinitions(URL web,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws
+ URL secureWebAPI,
+ ServiceInstanceData instanceData, ServiceEntry serviceEntry) throws
IOException {
- super.applyInitialRegistryDefinitions(web, secureWebAPI, instanceData);
+ super.applyInitialRegistryDefinitions(web, secureWebAPI, instanceData,
+ serviceEntry);
}
@Override
[04/11] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4058b9ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4058b9ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4058b9ab
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4058b9ab9deec38b9ecce1ef5c15eb0a4fe61022
Parents: 9ec1288 120c3b9
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 1 16:47:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 1 16:47:03 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 3 +-
app-packages/accumulo/pom.xml | 20 +-
.../command-logger/application-pkg/pom.xml | 7 +-
.../command-logger/slider-pkg/appConfig.json | 1 -
app-packages/command-logger/slider-pkg/pom.xml | 2 +-
app-packages/hbase/README.txt | 10 +-
app-packages/hbase/appConfig.json | 11 +-
app-packages/hbase/get-hbase-site.sh | 25 +
app-packages/hbase/pom.xml | 154 ++++-
app-packages/hbase/resources.json | 8 +-
app-packages/hbase/src/assembly/hbase.xml | 1 +
.../hbase/HBaseAgentCommandTestBase.groovy | 52 ++
.../slider/funtest/hbase/HBaseBasicIT.groovy | 114 ++++
.../funtest/hbase/HBaseMonitorSSLIT.groovy | 73 +++
.../hbase/StubToForceGroovyTestsToCompile.java | 22 +
.../test/resources/appConfig_monitor_ssl.json | 68 +++
.../hbase/src/test/resources/resources.json | 21 +
app-packages/memcached-win/README.txt | 36 ++
app-packages/memcached-win/appConfig.json | 22 +
app-packages/memcached-win/metainfo.xml | 51 ++
.../jmemcached-cli-1.0.0.jar.REPLACEME | 16 +
.../jmemcached-core-1.0.0.jar.REPLACEME | 16 +
.../memcached-win/package/scripts/memcached.py | 54 ++
.../memcached-win/package/scripts/params.py | 32 +
app-packages/memcached-win/resources.json | 16 +
app-packages/memcached/README.txt | 35 ++
app-packages/memcached/appConfig.json | 22 +
app-packages/memcached/metainfo.xml | 51 ++
.../package/files/jmemcached-1.0.0.tar.REPLACE | 20 +
.../memcached/package/scripts/memcached.py | 53 ++
.../memcached/package/scripts/params.py | 32 +
app-packages/memcached/resources.json | 16 +
app-packages/storm-v0_91/README.txt | 33 --
app-packages/storm-v0_91/appConfig.json | 126 ----
.../storm-v0_91/configuration/global.xml | 39 --
.../storm-v0_91/configuration/storm-site.xml | 587 -------------------
app-packages/storm-v0_91/ganglia_metrics.json | 31 -
app-packages/storm-v0_91/jmx_metrics.json | 31 -
app-packages/storm-v0_91/metainfo.xml | 145 -----
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 -
.../storm-v0_91/package/scripts/drpc_server.py | 55 --
.../storm-v0_91/package/scripts/nimbus.py | 55 --
.../storm-v0_91/package/scripts/params.py | 58 --
.../storm-v0_91/package/scripts/rest_api.py | 57 --
.../storm-v0_91/package/scripts/service.py | 95 ---
.../package/scripts/status_params.py | 36 --
.../storm-v0_91/package/scripts/storm.py | 50 --
.../storm-v0_91/package/scripts/supervisor.py | 61 --
.../storm-v0_91/package/scripts/ui_server.py | 55 --
.../storm-v0_91/package/scripts/yaml_config.py | 69 ---
.../package/templates/config.yaml.j2 | 48 --
.../package/templates/storm_jaas.conf.j2 | 26 -
app-packages/storm-v0_91/resources.json | 31 -
app-packages/storm/README.txt | 37 ++
app-packages/storm/appConfig.json | 126 ++++
app-packages/storm/configuration/global.xml | 39 ++
app-packages/storm/configuration/storm-site.xml | 587 +++++++++++++++++++
app-packages/storm/ganglia_metrics.json | 31 +
app-packages/storm/jmx_metrics.json | 31 +
app-packages/storm/metainfo.xml | 145 +++++
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 +
.../storm/package/scripts/drpc_server.py | 55 ++
app-packages/storm/package/scripts/nimbus.py | 55 ++
app-packages/storm/package/scripts/params.py | 58 ++
app-packages/storm/package/scripts/rest_api.py | 57 ++
app-packages/storm/package/scripts/service.py | 95 +++
.../storm/package/scripts/status_params.py | 36 ++
app-packages/storm/package/scripts/storm.py | 50 ++
.../storm/package/scripts/supervisor.py | 61 ++
app-packages/storm/package/scripts/ui_server.py | 55 ++
.../storm/package/scripts/yaml_config.py | 69 +++
.../storm/package/templates/config.yaml.j2 | 48 ++
.../storm/package/templates/storm_jaas.conf.j2 | 26 +
app-packages/storm/resources.json | 31 +
pom.xml | 70 ++-
slider-agent/conf/agent.ini | 6 +-
slider-agent/pom.xml | 6 +-
.../src/main/python/agent/AgentConfig.py | 12 +-
.../src/main/python/agent/Controller.py | 32 +-
.../python/agent/CustomServiceOrchestrator.py | 38 +-
slider-agent/src/main/python/agent/NetUtil.py | 8 +-
.../src/main/python/agent/ProcessHelper.py | 3 +-
.../src/main/python/agent/PythonExecutor.py | 7 +-
slider-agent/src/main/python/agent/main.py | 44 +-
slider-agent/src/main/python/agent/security.py | 7 +-
slider-agent/src/main/python/agent/shell.py | 102 +++-
.../python/resource_management/core/logger.py | 5 +-
.../core/providers/__init__.py | 5 +
.../core/providers/accounts.py | 20 +-
.../core/providers/system.py | 28 +-
.../core/providers/windows/__init__.py | 20 +
.../core/providers/windows/service.py | 65 ++
.../core/providers/windows/system.py | 142 +++++
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/os_check.py | 166 ++++--
.../libraries/providers/__init__.py | 2 +
.../libraries/providers/monitor_webserver.py | 2 +-
.../libraries/script/script.py | 15 +-
.../src/test/python/agent/TestActionQueue.py | 2 +-
.../src/test/python/agent/TestController.py | 9 +-
slider-agent/src/test/python/agent/TestMain.py | 26 +-
.../src/test/python/agent/TestNetUtil.py | 2 +-
.../src/test/python/agent/TestRegistration.py | 6 +-
slider-agent/src/test/python/agent/TestShell.py | 33 +-
.../TestDirectoryResource.py | 2 +-
.../resource_management/TestGroupResource.py | 138 -----
.../TestGroupResource.py.disabled | 138 +++++
.../resource_management/TestUserResource.py | 192 ------
.../TestUserResource.py.disabled | 192 ++++++
slider-assembly/pom.xml | 185 +++++-
slider-assembly/src/conf-hdp/log4j.properties | 58 ++
slider-assembly/src/conf-hdp/slider-client.xml | 77 +++
slider-assembly/src/conf/slider-client.xml | 3 -
slider-core/pom.xml | 17 +-
.../java/org/apache/slider/api/StatusKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 141 ++++-
.../org/apache/slider/common/SliderKeys.java | 14 +
.../common/SliderXMLConfKeysForTesting.java | 5 +
.../slider/common/params/ActionUpdateArgs.java | 32 +
.../apache/slider/common/params/ClientArgs.java | 9 +
.../slider/common/params/SliderActions.java | 3 +
.../slider/common/tools/CoreFileSystem.java | 1 +
.../apache/slider/common/tools/SliderUtils.java | 8 +-
.../slider/core/build/InstanceBuilder.java | 6 +-
.../slider/core/exceptions/ErrorStrings.java | 2 +-
.../core/launch/ClasspathConstructor.java | 1 -
.../slider/core/main/ServiceLauncher.java | 7 +-
.../registry/info/CustomRegistryConstants.java | 2 +-
.../slider/core/zk/BlockingZKWatcher.java | 8 +-
.../apache/slider/core/zk/ZKIntegration.java | 23 +-
.../providers/AbstractProviderService.java | 7 +-
.../slider/providers/ProviderService.java | 8 +-
.../providers/agent/AgentClientProvider.java | 16 +-
.../slider/providers/agent/AgentKeys.java | 6 +-
.../providers/agent/AgentProviderService.java | 80 ++-
.../slideram/SliderAMClientProvider.java | 7 +-
.../slideram/SliderAMProviderService.java | 18 +-
.../server/appmaster/SliderAppMaster.java | 73 ++-
.../server/appmaster/web/AgentService.java | 53 ++
.../server/appmaster/web/SliderAmIpFilter.java | 18 +-
.../slider/server/appmaster/web/WebAppApi.java | 9 +-
.../server/appmaster/web/WebAppApiImpl.java | 16 +-
.../appmaster/web/rest/AMWebServices.java | 7 +-
.../server/appmaster/web/rest/RestPaths.java | 5 +-
.../appmaster/web/rest/agent/AgentResource.java | 25 +-
.../appmaster/web/rest/agent/AgentWebApp.java | 203 +++++++
.../web/rest/agent/AgentWebServices.java | 40 ++
.../services/security/CertificateManager.java | 257 ++++++++
.../server/services/security/SecurityUtils.java | 225 +++++++
.../services/security/SignCertResponse.java | 67 +++
.../server/services/security/SignMessage.java | 54 ++
.../services/workflow/LongLivedProcess.java | 78 +--
.../providers/slideram/instance/appconf.json | 5 +-
.../main/resources/webapps/slideragent/.keep | 0
.../standalone/TestBuildStandaloneAM.groovy | 60 ++
.../slider/client/TestCommonArgParsing.groovy | 13 +
.../common/tools/TestZKIntegration.groovy | 53 ++
.../slider/providers/agent/AgentTestBase.groovy | 30 +
.../agent/TestAgentAMManagementWS.groovy | 56 +-
.../providers/agent/TestBuildBasicAgent.groovy | 150 ++++-
.../model/mock/MockProviderService.groovy | 12 +-
.../view/TestClusterSpecificationBlock.groovy | 2 +-
.../web/view/TestContainerStatsBlock.groovy | 2 +-
.../appmaster/web/view/TestIndexBlock.groovy | 2 +-
.../apache/slider/test/SliderTestBase.groovy | 5 +
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../agent/TestAgentProviderService.java | 52 +-
.../web/rest/agent/TestAMAgentWebServices.java | 217 +++----
.../management/TestAMManagementWebServices.java | 2 +-
.../security/TestCertificateManager.java | 60 ++
slider-core/src/test/python/agent.py | 17 +-
slider-core/src/test/python/agent/main.py | 29 +-
slider-core/src/test/python/echo.py | 21 +-
slider-funtest/pom.xml | 29 +-
.../abstracttests/AbstractTestBuildSetup.groovy | 3 -
.../framework/AgentCommandTestBase.groovy | 47 --
.../funtest/framework/CommandTestBase.groovy | 57 +-
.../funtest/framework/FuntestProperties.groovy | 3 -
.../accumulo/StubToForceGroovySrcToCompile.java | 22 -
.../slider/funtest/basic/BuildSetupIT.groovy | 36 ++
.../funtest/basic/ClusterConnectivityIT.groovy | 104 ++++
.../funtest/basic/SignCorrectionIT.groovy | 43 ++
.../slider/funtest/basic/TestBuildSetup.groovy | 36 --
.../basic/TestClusterConnectivity.groovy | 109 ----
.../funtest/basic/TestSignCorrection.groovy | 43 --
.../funtest/commands/ListCommandIT.groovy | 36 ++
.../funtest/commands/SimpleCommandsIT.groovy | 45 ++
.../funtest/commands/TestListCommand.groovy | 41 --
.../funtest/commands/TestSimpleCommands.groovy | 45 --
.../TestUnknownClusterOperations.groovy | 105 ----
.../commands/UnknownClusterOperationsIT.groovy | 100 ++++
.../lifecycle/AgentClusterLifecycleIT.groovy | 192 ++++++
.../funtest/lifecycle/AgentFailures2IT.groovy | 104 ++++
.../funtest/lifecycle/AgentFailuresIT.groovy | 104 ++++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 84 +++
.../lifecycle/ClusterBuildDestroyIT.groovy | 82 +++
.../lifecycle/TestAgentClusterLifecycle.groovy | 192 ------
.../funtest/lifecycle/TestAgentFailures.groovy | 104 ----
.../funtest/lifecycle/TestAgentFailures2.groovy | 104 ----
.../lifecycle/TestAppsThroughAgent.groovy | 84 ---
.../lifecycle/TestClusterBuildDestroy.groovy | 82 ---
slider-install/pom.xml | 111 ++++
.../accumulo/accumulo-funtests/pom.xml | 29 +-
.../slider/providers/accumulo/TestStub.groovy | 32 -
.../funtest/AccumuloBuildSetupIT.groovy | 37 ++
.../accumulo/funtest/AccumuloCIIT.groovy | 109 ++++
.../funtest/AccumuloCommandTestBase.groovy | 53 +-
.../funtest/FunctionalAccumuloClusterIT.groovy | 145 +++++
.../FunctionalAccumuloM1T1GC1Mon1IT.groovy | 57 ++
.../funtest/TestAccumuloBuildSetup.groovy | 38 --
.../accumulo/funtest/TestAccumuloCI.groovy | 115 ----
.../TestFunctionalAccumuloCluster.groovy | 146 -----
.../TestFunctionalAccumuloM1T1GC1Mon1.groovy | 59 --
.../StubToForceGroovyTestsToCompile.java | 22 +
.../accumulo/slider-accumulo-provider/pom.xml | 10 +-
slider-providers/hbase/hbase-funtests/pom.xml | 29 +-
.../slider/providers/hbase/TestStub.groovy | 32 -
.../hbase/funtest/DemoHBaseCluster.groovy | 2 +-
.../funtest/FunctionalHBaseClusterIT.groovy | 223 +++++++
.../hbase/funtest/HBaseBuildSetupIT.groovy | 36 ++
.../funtest/HBaseClusterBuildDestroyIT.groovy | 87 +++
.../funtest/HBaseClusterLifecycleIT.groovy | 183 ++++++
.../hbase/funtest/HBaseCommandTestBase.groovy | 1 -
.../hbase/funtest/HBaseIntegrationIT.groovy | 69 +++
.../providers/hbase/funtest/HBaseLoadIT.groovy | 64 ++
.../hbase/funtest/HBaseNodeFailureIT.groovy | 135 +++++
.../providers/hbase/funtest/ImagesIT.groovy | 64 ++
.../funtest/TestFunctionalHBaseCluster.groovy | 223 -------
.../hbase/funtest/TestHBaseBuildSetup.groovy | 37 --
.../funtest/TestHBaseClusterBuildDestroy.groovy | 88 ---
.../funtest/TestHBaseClusterLifecycle.groovy | 183 ------
.../hbase/funtest/TestHBaseIntegration.groovy | 69 ---
.../hbase/funtest/TestHBaseLoad.groovy | 64 --
.../hbase/funtest/TestHBaseNodeFailure.groovy | 135 -----
.../providers/hbase/funtest/TestImages.groovy | 65 --
.../hbase/slider-hbase-provider/pom.xml | 20 +-
.../providers/hbase/HBaseClientProvider.java | 2 +
.../slider/providers/hbase/HBaseKeys.java | 4 +
.../providers/hbase/HBaseProviderService.java | 10 +-
.../slider/providers/hbase/HBaseRoles.java | 3 +
src/test/clusters/c6401/slider/log4j.properties | 30 -
.../clusters/morzine/slider/log4j.properties | 30 -
.../clusters/offline/slider/log4j.properties | 30 -
.../clusters/offline/slider/slider-client.xml | 11 -
.../clusters/remote/slider/log4j.properties | 30 -
.../clusters/sandbox/slider/log4j.properties | 30 -
.../ubuntu-secure/slider/log4j.properties | 1 -
247 files changed, 8555 insertions(+), 5129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4058b9ab/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 61e0297,7cca66c..0509964
--- a/pom.xml
+++ b/pom.xml
@@@ -115,9 -117,9 +117,9 @@@
<!--
core artifacts
-->
- <hadoop.version>2.4.1</hadoop.version>
+ <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
- <hbase.version>0.98.3-hadoop2</hbase.version>
+ <hbase.version>0.98.4-hadoop2</hbase.version>
<accumulo.version>1.6.0</accumulo.version>
<!--
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4058b9ab/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
[02/11] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/edd96e60
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/edd96e60
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/edd96e60
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: edd96e600588c3bc30bed6a49d42f31119154441
Parents: cfc6e1d a613e20
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jul 7 15:30:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jul 7 15:30:00 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 4 +-
app-packages/accumulo/pom.xml | 107 ++++++++-
app-packages/accumulo/resources.json | 15 +-
app-packages/accumulo/src/assembly/accumulo.xml | 1 +
.../AccumuloAgentCommandTestBase.groovy | 52 ++++
.../funtest/accumulo/AccumuloBasicIT.groovy | 117 +++++++++
.../accumulo/AccumuloMonitorSSLIT.groovy | 73 ++++++
.../funtest/accumulo/AccumuloReadWriteIT.groovy | 116 +++++++++
.../StubToForceGroovyTestsToCompile.java | 22 ++
.../test/resources/appConfig_monitor_ssl.json | 62 +++++
.../accumulo/src/test/resources/resources.json | 36 +++
app-packages/hbase/README.txt | 3 +
pom.xml | 2 +-
.../apache/slider/common/SliderXmlConfKeys.java | 2 +
.../slider/core/main/ServiceLauncher.java | 109 +++++++--
.../server/appmaster/SliderAppMaster.java | 46 +++-
.../slider/server/appmaster/state/AppState.java | 2 +-
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../server/appmaster/web/SliderAmIpFilter.java | 1 +
.../server/services/curator/CuratorService.java | 2 +-
.../resources/webapps/static/yarn.dt.plugins.js | 4 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 2 +-
slider-funtest/pom.xml | 5 +-
.../framework/AgentCommandTestBase.groovy | 237 +++++++++++++++++++
.../accumulo/StubToForceGroovySrcToCompile.java | 22 ++
.../basic/TestClusterConnectivity.groovy | 35 ++-
.../lifecycle/AgentCommandTestBase.groovy | 233 ------------------
.../lifecycle/TestAgentClusterLifecycle.groovy | 1 +
.../funtest/lifecycle/TestAgentFailures.groovy | 1 +
.../funtest/lifecycle/TestAgentFailures2.groovy | 1 +
.../lifecycle/TestAppsThroughAgent.groovy | 1 +
.../lifecycle/TestClusterBuildDestroy.groovy | 1 +
.../accumulo/AccumuloProviderService.java | 4 +
33 files changed, 1025 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/edd96e60/pom.xml
----------------------------------------------------------------------
[05/11] git commit: SLIDER-149 yarn-registry dependency
Posted by st...@apache.org.
SLIDER-149 yarn-registry dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/adc4640e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/adc4640e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/adc4640e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: adc4640e5154964efe6948bf6788123ea9c294e0
Parents: 4058b9a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 1 19:52:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 1 19:52:39 2014 +0100
----------------------------------------------------------------------
pom.xml | 6 ++++++
slider-core/pom.xml | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/adc4640e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0509964..b2670b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -518,6 +518,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<version>${hadoop.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/adc4640e/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index a44e3bc..19e539e 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -306,6 +306,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>