You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/21 16:42:48 UTC
[12/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b86aff8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b86aff8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b86aff8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0b86aff8de6910005b99f159d6807888895971ba
Parents: 34396a0 52f1f6f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 15:14:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 15:24:15 2014 +0100
----------------------------------------------------------------------
pom.xml | 9 +-
slider-agent/pom.xml | 4 +-
.../src/main/python/agent/AgentConfig.py | 2 +
slider-agent/src/main/python/agent/Constants.py | 4 +-
.../src/main/python/agent/Controller.py | 52 +-
slider-agent/src/main/python/agent/Heartbeat.py | 6 +-
slider-agent/src/main/python/agent/Registry.py | 57 +
slider-agent/src/main/python/agent/main.py | 29 +-
slider-agent/src/main/python/kazoo/__init__.py | 1 +
slider-agent/src/main/python/kazoo/client.py | 1412 ++++++++++++++++++
.../src/main/python/kazoo/exceptions.py | 199 +++
.../src/main/python/kazoo/handlers/__init__.py | 1 +
.../src/main/python/kazoo/handlers/gevent.py | 161 ++
.../src/main/python/kazoo/handlers/threading.py | 287 ++++
.../src/main/python/kazoo/handlers/utils.py | 93 ++
slider-agent/src/main/python/kazoo/hosts.py | 26 +
.../src/main/python/kazoo/interfaces.py | 203 +++
.../src/main/python/kazoo/loggingsupport.py | 2 +
.../src/main/python/kazoo/protocol/__init__.py | 1 +
.../main/python/kazoo/protocol/connection.py | 623 ++++++++
.../src/main/python/kazoo/protocol/paths.py | 54 +
.../main/python/kazoo/protocol/serialization.py | 396 +++++
.../src/main/python/kazoo/protocol/states.py | 237 +++
.../src/main/python/kazoo/recipe/__init__.py | 1 +
.../src/main/python/kazoo/recipe/barrier.py | 214 +++
.../src/main/python/kazoo/recipe/counter.py | 94 ++
.../src/main/python/kazoo/recipe/election.py | 79 +
.../src/main/python/kazoo/recipe/lock.py | 520 +++++++
.../src/main/python/kazoo/recipe/partitioner.py | 377 +++++
.../src/main/python/kazoo/recipe/party.py | 118 ++
.../src/main/python/kazoo/recipe/queue.py | 321 ++++
.../src/main/python/kazoo/recipe/watchers.py | 419 ++++++
slider-agent/src/main/python/kazoo/retry.py | 150 ++
slider-agent/src/main/python/kazoo/security.py | 138 ++
.../src/main/python/kazoo/testing/__init__.py | 5 +
.../src/main/python/kazoo/testing/common.py | 283 ++++
.../src/main/python/kazoo/testing/harness.py | 180 +++
.../src/main/python/kazoo/tests/__init__.py | 0
.../src/main/python/kazoo/tests/test_barrier.py | 157 ++
.../src/main/python/kazoo/tests/test_build.py | 29 +
.../src/main/python/kazoo/tests/test_client.py | 1098 ++++++++++++++
.../main/python/kazoo/tests/test_connection.py | 319 ++++
.../src/main/python/kazoo/tests/test_counter.py | 35 +
.../main/python/kazoo/tests/test_election.py | 139 ++
.../main/python/kazoo/tests/test_exceptions.py | 22 +
.../python/kazoo/tests/test_gevent_handler.py | 160 ++
.../src/main/python/kazoo/tests/test_lock.py | 517 +++++++
.../main/python/kazoo/tests/test_partitioner.py | 92 ++
.../src/main/python/kazoo/tests/test_party.py | 84 ++
.../src/main/python/kazoo/tests/test_paths.py | 98 ++
.../src/main/python/kazoo/tests/test_queue.py | 179 +++
.../src/main/python/kazoo/tests/test_retry.py | 77 +
.../main/python/kazoo/tests/test_security.py | 40 +
.../kazoo/tests/test_threading_handler.py | 326 ++++
.../main/python/kazoo/tests/test_watchers.py | 489 ++++++
.../src/main/python/kazoo/tests/util.py | 126 ++
.../src/test/python/agent/TestHeartbeat.py | 12 +-
slider-agent/src/test/python/agent/TestMain.py | 16 +-
slider-assembly/src/main/bash/README.md | 102 ++
slider-assembly/src/main/bash/slider-client.xml | 83 +
slider-assembly/src/main/bash/slider_destroy | 64 +
slider-assembly/src/main/bash/slider_setup | 173 +++
slider-assembly/src/main/bash/slider_setup.conf | 60 +
slider-core/pom.xml | 46 +-
.../api/ClusterDescriptionOperations.java | 12 +-
.../org/apache/slider/api/InternalKeys.java | 137 ++
.../java/org/apache/slider/api/OptionKeys.java | 92 +-
.../org/apache/slider/api/ResourceKeys.java | 62 +
.../org/apache/slider/client/SliderClient.java | 17 +-
.../apache/slider/common/SliderExitCodes.java | 2 +-
.../org/apache/slider/common/SliderKeys.java | 5 +
.../apache/slider/common/SliderXmlConfKeys.java | 13 -
.../apache/slider/common/tools/SliderUtils.java | 23 +-
.../slider/core/build/InstanceBuilder.java | 3 +-
.../apache/slider/core/conf/MapOperations.java | 54 +-
.../providers/AbstractProviderService.java | 18 +
.../slider/providers/ProviderService.java | 22 +
.../apache/slider/providers/ProviderUtils.java | 9 +-
.../providers/agent/AgentClientProvider.java | 8 +-
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 134 +-
.../slideram/SliderAMClientProvider.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 231 ++-
.../server/appmaster/actions/ActionHalt.java | 9 +-
.../appmaster/actions/ActionKillContainer.java | 67 +
.../appmaster/actions/ActionStartContainer.java | 15 +-
.../appmaster/actions/ActionStopQueue.java | 15 +-
.../appmaster/actions/ActionStopSlider.java | 15 +-
.../server/appmaster/actions/AsyncAction.java | 16 +-
.../actions/ProviderReportedContainerLoss.java | 7 +-
.../actions/ProviderStartupCompleted.java | 7 +-
.../server/appmaster/actions/QueueAccess.java | 31 +-
.../server/appmaster/actions/QueueExecutor.java | 7 +-
.../server/appmaster/actions/QueueService.java | 117 +-
.../appmaster/actions/RenewingAction.java | 35 +-
.../appmaster/actions/ResetFailureWindow.java | 39 +
.../server/appmaster/monkey/ChaosEntry.java | 86 ++
.../server/appmaster/monkey/ChaosKillAM.java | 48 +
.../appmaster/monkey/ChaosKillContainer.java | 75 +
.../appmaster/monkey/ChaosMonkeyService.java | 86 ++
.../server/appmaster/monkey/ChaosTarget.java | 24 +
.../appmaster/monkey/MonkeyPlayAction.java | 48 +
.../slider/server/appmaster/state/AppState.java | 125 +-
.../server/appmaster/state/RoleHistory.java | 2 +-
.../server/appmaster/state/RoleInstance.java | 4 +-
.../server/appmaster/state/RoleStatus.java | 24 +-
.../appmaster/web/rest/agent/HeartBeat.java | 11 +
.../services/workflow/LongLivedProcess.java | 7 +-
.../workflow/WorkflowExecutorService.java | 2 +-
.../TestFreezeThawMasterlessAM.groovy | 5 +-
.../standalone/TestBuildStandaloneAM.groovy | 13 +-
.../TestStandaloneAMMonkeyRestart.groovy | 94 ++
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
.../server/appmaster/actions/TestActions.groovy | 43 +-
.../TestMockAppStateContainerFailure.groovy | 46 +-
.../TestMockAppStateDynamicRoles.groovy | 2 +-
.../TestMockAppStateFlexDynamicRoles.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 2 +-
.../TestMockAppStateRMOperations.groovy | 2 +-
.../TestMockAppStateRebuildOnAMRestart.groovy | 2 +-
.../TestMockAppStateRolePlacement.groovy | 2 +-
.../appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../TestMockContainerResourceAllocations.groovy | 2 +-
.../model/mock/BaseMockAppStateTest.groovy | 22 +-
.../model/mock/MockProviderService.groovy | 12 +
.../model/mock/MockRMOperationHandler.groovy | 7 +-
.../model/monkey/TestMockMonkey.groovy | 160 ++
.../apache/slider/test/SliderTestBase.groovy | 7 +-
.../apache/slider/test/SliderTestUtils.groovy | 4 +
.../agent/TestAgentProviderService.java | 35 +-
slider-core/src/test/python/agent/main.py | 5 +-
slider-core/src/test/resources/log4j.properties | 2 +-
slider-install/README.md | 102 --
slider-install/pom.xml | 111 --
slider-install/src/main/bash/slider-client.xml | 83 -
slider-install/src/main/bash/slider_destroy | 64 -
slider-install/src/main/bash/slider_setup | 173 ---
slider-install/src/main/bash/slider_setup.conf | 60 -
.../accumulo/AccumuloClientProvider.java | 3 +-
.../accumulo/AccumuloProviderService.java | 9 +-
.../providers/hbase/HBaseClientProvider.java | 3 +-
.../providers/hbase/HBaseProviderService.java | 4 +-
.../failures/TestFailureThreshold.groovy | 150 --
.../TestRegionServerFailureThreshold.groovy | 185 +++
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
.../src/test/resources/log4j.properties | 1 +
146 files changed, 13291 insertions(+), 1205 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 00f98cb,1337570..61f4d54
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@@ -33,11 -33,9 +33,12 @@@ import org.apache.hadoop.yarn.api.recor
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.registry.client.binding.zk.ZKPathDumper;
+import org.apache.hadoop.yarn.registry.server.services.YarnRegistryService;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
+ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b6f1e49,7825e24..5ef32bf
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@@ -196,9 -198,13 +203,14 @@@ public class SliderAppMaster extends Ab
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
+ public static final String SLIDER_AM_RPC = "Slider AM RPC";
public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
+ /**
+ * Singleton of metrics registry
+ */
+ public static final MetricRegistry metrics = new MetricRegistry();
+
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 711bc24,0665a9a..434f4c2
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@@ -64,11 -65,20 +65,17 @@@ class TestBuildStandaloneAM extends Age
//but the cluster is still there for the default
assert 0 == sliderClient.actionExists(clustername, false)
-
-
-
- // verify the YARN registry doesn't know of it
- def serviceRegistryClient = sliderClient.YARNRegistryClient
+
+ def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport report = serviceRegistryClient.findInstance(clustername)
assert report == null;
-
+
+ // verify that global resource options propagate from the CLI
+ def aggregateConf = sliderClient.loadPersistedClusterDescription(clustername)
+ def windowDays = aggregateConf.resourceOperations.globalOptions.getMandatoryOptionInt(
+ "yarn.container.failure.window.years")
+ assert 4 == windowDays
+
//and a second attempt will fail as the cluster now exists
try {
ServiceLauncher<SliderClient> cluster2 = createOrBuildCluster(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b86aff8/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --cc slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index a4111fa,feffb08..747ccfa
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@@ -22,11 -22,9 +22,11 @@@ package org.apache.slider.providers.hba
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.registry.client.api.RegistryWriter;
+import org.apache.hadoop.yarn.registry.client.types.ServiceEntry;
+ import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
- import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.core.conf.AggregateConf;