You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/05/20 20:27:21 UTC
svn commit: r1596353 [2/2] - in /incubator/slider/trunk:
slider-core/src/main/java/org/apache/slider/
slider-core/src/main/java/org/apache/slider/common/tools/
slider-core/src/main/java/org/apache/slider/core/registry/docstore/
slider-core/src/main/jav...
Modified: incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java?rev=1596353&r1=1596352&r2=1596353&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java (original)
+++ incubator/slider/trunk/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java Tue May 20 18:27:20 2014
@@ -126,7 +126,6 @@ public class AccumuloProviderService ext
Path containerTmpDirPath) throws IOException, SliderException {
this.fileSystem = fileSystem;
- this.instanceDefinition = instanceDefinition;
// Set the environment
launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
Modified: incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java?rev=1596353&r1=1596352&r2=1596353&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java (original)
+++ incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java Tue May 20 18:27:20 2014
@@ -67,8 +67,11 @@ public interface HBaseKeys {
String HBASE_LOG_DIR = "HBASE_LOG_DIR";
String HBASE_HEAPSIZE = "HBASE_HEAPSIZE";
- String HBASE_GC_OPTS = "SERVER_GC_OPTS";
+ String HBASE_GC_OPTS = "SERVER_GC_OPTS";
+ String PROPAGATED_CONFDIR = "PROPAGATED_CONFDIR";
+ String HBASE_SERVICE_TYPE = "org.apache.hbase";
+ String HBASE_SITE_PUBLISHED_CONFIG = "hbase-site";
}
Modified: incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java?rev=1596353&r1=1596352&r2=1596353&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java (original)
+++ incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java Tue May 20 18:27:20 2014
@@ -34,6 +34,9 @@ import org.apache.slider.core.launch.Con
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
@@ -58,6 +61,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER;
+
/**
* This class implements the server-side aspects
* of an HBase Cluster
@@ -73,7 +78,6 @@ public class HBaseProviderService extend
private static final ProviderUtils providerUtils = new ProviderUtils(log);
private HBaseClientProvider clientProvider;
private Configuration siteConf;
- private SliderFileSystem sliderFileSystem = null;
public HBaseProviderService() {
super("HBaseProviderService");
@@ -101,7 +105,7 @@ public class HBaseProviderService extend
/**
* Validate the cluster specification. This can be invoked on both
* server and client
- * @param instanceDefinition
+ * @param instanceDefinition the instance definition to validate
*/
@Override // Client and Server
public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
@@ -114,20 +118,18 @@ public class HBaseProviderService extend
AggregateConf instanceDefinition,
Container container,
String role,
- SliderFileSystem sliderFileSystem,
+ SliderFileSystem coreFS,
Path generatedConfPath,
MapOperations resourceComponent,
MapOperations appComponent,
Path containerTmpDirPath) throws IOException, SliderException {
- this.sliderFileSystem = sliderFileSystem;
- this.instanceDefinition = instanceDefinition;
// Set the environment
launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
launcher.setEnv(HBASE_LOG_DIR, providerUtils.getLogdir());
- launcher.setEnv("PROPAGATED_CONFDIR",
+ launcher.setEnv(PROPAGATED_CONFDIR,
ProviderUtils.convertToAppRelativePath(
SliderKeys.PROPAGATED_CONF_DIR_NAME) );
@@ -135,13 +137,14 @@ public class HBaseProviderService extend
//local resources
//add the configuration resources
- launcher.addLocalResources(sliderFileSystem.submitDirectory(
+ launcher.addLocalResources(coreFS.submitDirectory(
generatedConfPath,
SliderKeys.PROPAGATED_CONF_DIR_NAME));
//Add binaries
//now add the image if it was set
- String imageURI = instanceDefinition.getInternalOperations().get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- sliderFileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);
+ String imageURI = instanceDefinition.getInternalOperations()
+ .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ coreFS.maybeAddImagePath(launcher.getLocalResources(), imageURI);
CommandLineBuilder cli = new CommandLineBuilder();
@@ -191,6 +194,44 @@ public class HBaseProviderService extend
}
+ @Override
+ public void applyInitialRegistryDefinitions(URL web,
+ ServiceInstanceData instanceData) throws
+ IOException {
+ super.applyInitialRegistryDefinitions(web, instanceData);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ registerHBaseServiceEntry();
+
+
+ super.serviceStart();
+ }
+
+ private void registerHBaseServiceEntry() throws IOException {
+
+ // not a URL, but needed
+ URL hbaseURL = new URL("http://localhost:0");
+ ServiceInstanceData instanceData = new ServiceInstanceData();
+ PublishedConfiguration publishedSite =
+ new PublishedConfiguration("HBase site",
+ siteConf);
+ PublishedConfigSet configSet =
+ amState.getOrCreatePublishedConfigSet(HBASE_SERVICE_TYPE);
+ instanceData.externalView.configurationsURL = SliderUtils.appendToURL(
+ amWebAPI.toExternalForm(), SLIDER_PATH_PUBLISHER, HBASE_SERVICE_TYPE);
+ configSet.put(HBASE_SITE_PUBLISHED_CONFIG,
+ publishedSite);
+ String name = amState.getApplicationName()+".hbase";
+ log.info("registering {}/{}", name, HBASE_SERVICE_TYPE);
+ registry.registerServiceInstance(HBASE_SERVICE_TYPE,
+ name,
+ null,
+ instanceData
+ );
+ }
+
/**
* Run this service
*
@@ -292,4 +333,6 @@ public class HBaseProviderService extend
response.setResponseId(id + 1L);
return response;
}
+
+
}
Modified: incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy?rev=1596353&r1=1596352&r2=1596353&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy (original)
+++ incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestHBaseMaster.groovy Tue May 20 18:27:20 2014
@@ -20,19 +20,20 @@ package org.apache.slider.providers.hbas
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.hbase.ClusterStatus
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.RoleKeys
-import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.info.ServiceInstanceData
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.core.registry.zk.ZKIntegration
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.server.services.curator.CuratorServiceInstance
+import org.apache.slider.server.services.registry.SliderRegistryService
import org.junit.Test
/**
@@ -41,6 +42,7 @@ import org.junit.Test
@CompileStatic
@Slf4j
class TestHBaseMaster extends HBaseMiniClusterTestBase {
+ public static final String HBASE_SITE = HBaseKeys.HBASE_SITE_PUBLISHED_CONFIG;
@Test
public void testHBaseMaster() throws Throwable {
@@ -57,22 +59,49 @@ class TestHBaseMaster extends HBaseMiniC
],
true,
true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
- ClusterDescription status = sliderClient.getClusterDescription(clustername)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+ ClusterDescription status = client.getClusterDescription(clustername)
//dumpFullHBaseConf(sliderClient, clustername)
- basicHBaseClusterStartupSequence(sliderClient)
+ basicHBaseClusterStartupSequence(client)
//verify the #of region servers is as expected
- dumpClusterStatus(sliderClient, "post-hbase-boot status")
+ dumpClusterStatus(client, "post-hbase-boot status")
//get the hbase status
- waitForHBaseRegionServerCount(sliderClient, clustername, 1, hbaseClusterStartupToLiveTime)
- waitForWorkerInstanceCount(sliderClient, 1, hbaseClusterStartupToLiveTime)
- waitForRoleCount(sliderClient, HBaseKeys.ROLE_MASTER, 1,
+ waitForHBaseRegionServerCount(client, clustername, 1, hbaseClusterStartupToLiveTime)
+ waitForWorkerInstanceCount(client, 1, hbaseClusterStartupToLiveTime)
+ waitForRoleCount(client, HBaseKeys.ROLE_MASTER, 1,
hbaseClusterStartupToLiveTime)
+
+ // look up the registry entries for HBase
+ describe "service registry names"
+ SliderRegistryService registryService = client.registry
+ def names = registryService.getServiceTypes();
+ dumpRegistryNames(names)
+
+ List<CuratorServiceInstance<ServiceInstanceData>> instances =
+ client.listRegistryInstances();
+ def hbaseService = registryService.findByID(
+ instances,
+ HBaseKeys.HBASE_SERVICE_TYPE)
+ assert hbaseService
+ RegistryRetriever retriever = new RegistryRetriever(hbaseService.payload)
+ log.info retriever.toString()
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[HBASE_SITE]
+
+
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ HBASE_SITE,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
}
}
Modified: incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy?rev=1596353&r1=1596352&r2=1596353&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy (original)
+++ incubator/slider/trunk/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/live/TestTwoLiveClusters.groovy Tue May 20 18:27:20 2014
@@ -82,7 +82,7 @@ class TestTwoLiveClusters extends HBaseM
// registry instances def names = client.listRegistryNames(clustername)
describe "service registry names"
SliderRegistryService registry = cluster2Client.registry
- def names = registry.queryForNames();
+ def names = registry.getServiceTypes();
dumpRegistryNames(names)
List<String> instanceIds = sliderClient.listRegistryInstanceIDs()