You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/09 00:53:23 UTC
[34/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c51f2f0e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c51f2f0e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c51f2f0e
Branch: refs/heads/develop
Commit: c51f2f0eac6a7938e1f0acc27a4f5df90bb88339
Parents: 733745e 70b4b75
Author: Steve Loughran <st...@apache.org>
Authored: Sun Oct 5 19:56:47 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Oct 5 19:56:47 2014 -0700
----------------------------------------------------------------------
.../accumulo/configuration/accumulo-site.xml | 19 +-
app-packages/accumulo/configuration/global.xml | 7 +-
app-packages/accumulo/pom.xml | 2 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 4 +-
.../slider/funtest/accumulo/CertUtil.java | 50 +++-
.../app-pkg-template/appConfig-default.json | 2 +-
.../command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/hbase/pom.xml | 2 +-
app-packages/memcached/appConfig-default.json | 2 +-
app-packages/storm-win/appConfig-default.json | 2 -
app-packages/storm-win/metainfo.xml | 12 +-
.../storm-win/package/scripts/params.py | 2 -
.../storm-win/package/scripts/service.py | 1 +
.../storm-win/package/scripts/status_params.py | 2 -
.../storm-win/package/templates/config.yaml.j2 | 9 -
app-packages/storm/appConfig-default.json | 4 +-
app-packages/storm/metainfo.xml | 23 +-
app-packages/storm/package/scripts/params.py | 2 -
.../storm/package/templates/config.yaml.j2 | 9 -
app-packages/storm/resources-default.json | 4 -
pom.xml | 2 +
slider-assembly/src/main/scripts/slider | 2 +-
.../org/apache/slider/client/SliderClient.java | 77 +++--
.../apache/slider/common/SliderXmlConfKeys.java | 4 +-
.../apache/slider/core/conf/AggregateConf.java | 20 ++
.../apache/slider/core/conf/MapOperations.java | 16 +-
.../slider/core/exceptions/SliderException.java | 6 +-
.../TriggerClusterTeardownException.java | 16 +-
.../providers/AbstractClientProvider.java | 9 +-
.../providers/AbstractProviderService.java | 21 +-
.../apache/slider/providers/ProviderCore.java | 9 +-
.../apache/slider/providers/ProviderUtils.java | 6 +-
.../providers/agent/AgentProviderService.java | 28 +-
.../providers/agent/ComponentInstanceState.java | 2 +-
.../slideram/SliderAMClientProvider.java | 28 ++
.../slideram/SliderAMProviderService.java | 8 -
.../apache/slider/server/appmaster/AMUtils.java | 5 +-
.../server/appmaster/SliderAppMaster.java | 298 +++++++++++--------
.../appmaster/actions/ActionStopQueue.java | 8 +-
.../appmaster/actions/ActionStopSlider.java | 116 +++++++-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../server/appmaster/actions/QueueExecutor.java | 10 +
.../server/appmaster/actions/QueueService.java | 22 +-
.../slider/server/appmaster/state/AppState.java | 13 +-
.../server/appmaster/state/RoleStatus.java | 20 ++
.../workflow/WorkflowSequenceService.java | 8 +-
.../agent/actions/TestActionStatus.groovy | 16 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 99 ++++++
.../TestFreezeThawMasterlessAM.groovy | 92 ------
.../standalone/TestBuildStandaloneAM.groovy | 22 +-
.../standalone/TestStandaloneAMDestroy.groovy | 20 +-
.../TestStandaloneAMMonkeyRestart.groovy | 12 +-
.../standalone/TestStandaloneAMRestart.groovy | 77 ++++-
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
.../slider/providers/agent/TestAgentEcho.groovy | 18 +-
.../TestMockAppStateContainerFailure.groovy | 1 -
.../slider/test/YarnMiniClusterTestBase.groovy | 37 ++-
.../accumulo/live/TestAccFreezeThaw.groovy | 7 +-
.../providers/hbase/HBaseProviderService.java | 23 --
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
61 files changed, 857 insertions(+), 490 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --cc app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 7ddaf9f,5ebce0a..fd8af9a
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@@ -167,11 -167,13 +167,12 @@@ class AccumuloBasicIT extends AccumuloA
public static String getMonitorUrl(SliderClient sliderClient, String clusterName) {
int tries = 5
+ Exception caught;
while (true) {
try {
- CuratorServiceInstance<ServiceInstanceData> instance =
- sliderClient.getRegistry().queryForInstance(SliderKeys.APP_TYPE, clusterName)
- ServiceInstanceData serviceInstanceData = instance.payload
- RegistryRetriever retriever = new RegistryRetriever(serviceInstanceData)
+ ServiceRecord instance =
+ sliderClient.lookupServiceRecord(SliderKeys.APP_TYPE, clusterName)
+ RegistryRetriever retriever = new RegistryRetriever(instance)
PublishedConfiguration configuration = retriever.retrieveConfiguration(
retriever.getConfigurations(true), "quicklinks", true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 2625d74,5676f3f..a259e42
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@@ -50,14 -51,9 +51,15 @@@ import org.apache.hadoop.yarn.client.ap
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+ import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.yarn.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@@ -357,8 -358,8 +362,9 @@@ public class SliderAppMaster extends Ab
private final QueueService actionQueues = new QueueService();
private String agentOpsUrl;
private String agentStatusUrl;
+ private YarnRegistryViewForProviders yarnRegistryOperations;
private FsDelegationTokenManager fsDelegationTokenManager;
+ private RegisterApplicationMasterResponse amRegistrationData;
/**
* Service Constructor
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 13c876e,a14a14d..b6ac657
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@@ -30,6 -30,6 +30,7 @@@ import org.apache.slider.common.params.
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
++import org.apache.slider.core.registry.YarnAppListClient
import org.junit.Test
import static org.apache.slider.common.params.Arguments.ARG_COMP_OPT
@@@ -66,7 -66,9 +67,8 @@@ class TestBuildStandaloneAM extends Age
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
- def serviceRegistryClient = sliderClient.yarnAppListClient
-
+ // verify the YARN registry doesn't know of it
- def serviceRegistryClient = sliderClient.YARNRegistryClient
++ YarnAppListClient serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c51f2f0e/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/build/TestBuildThawClusterM1W1.groovy
----------------------------------------------------------------------