You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2017/08/14 21:13:53 UTC
[15/15] hadoop git commit: YARN-6903. Yarn-native-service framework
core rewrite. Contributed by Jian He
YARN-6903. Yarn-native-service framework core rewrite. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/164c0c4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/164c0c4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/164c0c4c
Branch: refs/heads/yarn-native-services
Commit: 164c0c4c9346dd6486f1880c7ea70602f27270b3
Parents: f1a358e
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Aug 14 14:12:12 2017 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Mon Aug 14 14:12:12 2017 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 8 +-
.../api/impl/ApplicationApiService.java | 139 +--
.../dev-support/findbugs-exclude.xml | 8 +
.../hadoop-yarn-slider-core/pom.xml | 1 +
.../hadoop/yarn/service/ClientAMProtocol.java | 40 +
.../hadoop/yarn/service/ClientAMService.java | 132 +++
.../yarn/service/ContainerFailureTracker.java | 89 ++
.../yarn/service/ContainerLaunchService.java | 101 +++
.../hadoop/yarn/service/ServiceContext.java | 41 +
.../hadoop/yarn/service/ServiceMaster.java | 132 +++
.../hadoop/yarn/service/ServiceMonitor.java | 143 ++++
.../hadoop/yarn/service/ServiceScheduler.java | 641 ++++++++++++++
.../yarn/service/client/ClientAMProxy.java | 49 ++
.../hadoop/yarn/service/client/ServiceCLI.java | 98 +++
.../yarn/service/client/ServiceClient.java | 836 +++++++++++++++++++
.../client/params/AbstractActionArgs.java | 180 ++++
.../service/client/params/ActionBuildArgs.java | 33 +
.../service/client/params/ActionCreateArgs.java | 35 +
.../client/params/ActionDependencyArgs.java | 65 ++
.../client/params/ActionDestroyArgs.java | 37 +
.../service/client/params/ActionFlexArgs.java | 55 ++
.../yarn/service/client/params/ArgOps.java | 156 ++++
.../yarn/service/client/params/Arguments.java | 138 +++
.../yarn/service/client/params/ClientArgs.java | 383 +++++++++
.../yarn/service/client/params/CommonArgs.java | 294 +++++++
.../client/params/ComponentArgsDelegate.java | 54 ++
.../service/client/params/SliderAMArgs.java | 57 ++
.../client/params/SliderAMCreateAction.java | 77 ++
.../service/client/params/SliderActions.java | 106 +++
.../service/compinstance/ComponentInstance.java | 493 +++++++++++
.../compinstance/ComponentInstanceEvent.java | 58 ++
.../ComponentInstanceEventType.java | 27 +
.../compinstance/ComponentInstanceId.java | 91 ++
.../compinstance/ComponentInstanceState.java | 26 +
.../yarn/service/component/Component.java | 487 +++++++++++
.../yarn/service/component/ComponentEvent.java | 83 ++
.../service/component/ComponentEventType.java | 26 +
.../yarn/service/component/ComponentState.java | 25 +
.../yarn/service/conf/SliderExitCodes.java | 88 ++
.../hadoop/yarn/service/conf/SliderKeys.java | 195 +++++
.../yarn/service/conf/SliderXmlConfKeys.java | 191 +++++
.../yarn/service/conf/YarnServiceConfKeys.java | 27 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 91 ++
.../impl/pb/service/ClientAMProtocolPB.java | 29 +
.../service/ClientAMProtocolPBServiceImpl.java | 70 ++
.../yarn/service/metrics/ServiceMetrics.java | 101 +++
.../provider/AbstractClientProvider.java | 128 +++
.../provider/AbstractProviderService.java | 97 +++
.../yarn/service/provider/ProviderFactory.java | 77 ++
.../yarn/service/provider/ProviderService.java | 37 +
.../yarn/service/provider/ProviderUtils.java | 471 +++++++++++
.../defaultImpl/DefaultClientProvider.java | 46 +
.../defaultImpl/DefaultProviderFactory.java | 51 ++
.../defaultImpl/DefaultProviderService.java | 36 +
.../provider/docker/DockerClientProvider.java | 53 ++
.../service/provider/docker/DockerKeys.java | 30 +
.../provider/docker/DockerProviderFactory.java | 52 ++
.../provider/docker/DockerProviderService.java | 57 ++
.../provider/tarball/TarballClientProvider.java | 65 ++
.../tarball/TarballProviderFactory.java | 52 ++
.../tarball/TarballProviderService.java | 48 ++
.../timelineservice/ServiceMetricsSink.java | 102 +++
.../ServiceTimelineEntityType.java | 39 +
.../timelineservice/ServiceTimelineEvent.java | 34 +
.../ServiceTimelineMetricsConstants.java | 95 +++
.../ServiceTimelinePublisher.java | 428 ++++++++++
.../service/timelineservice/package-info.java | 27 +
.../yarn/service/utils/ServiceApiUtil.java | 449 ++++++++++
.../apache/slider/api/ServiceApiConstants.java | 2 +-
.../slider/api/SliderClusterProtocol.java | 2 +-
.../apache/slider/api/resource/Application.java | 4 +-
.../apache/slider/api/resource/Component.java | 3 +
.../slider/api/resource/ContainerState.java | 2 +-
.../apache/slider/api/resource/Resource.java | 7 +
.../org/apache/slider/client/ClientUtils.java | 2 +-
.../org/apache/slider/client/SliderClient.java | 167 +++-
.../apache/slider/client/SliderClientAPI.java | 4 +-
.../slider/client/SliderYarnClientImpl.java | 17 +-
.../apache/slider/common/SliderExitCodes.java | 88 --
.../org/apache/slider/common/SliderKeys.java | 195 -----
.../apache/slider/common/SliderXmlConfKeys.java | 190 -----
.../common/params/AbstractActionArgs.java | 179 ----
.../common/params/AbstractArgsDelegate.java | 2 +
.../AbstractClusterBuildingActionArgs.java | 2 +
.../common/params/ActionAMSuicideArgs.java | 4 +-
.../slider/common/params/ActionBuildArgs.java | 31 -
.../slider/common/params/ActionClientArgs.java | 4 +-
.../slider/common/params/ActionCreateArgs.java | 33 -
.../common/params/ActionDependencyArgs.java | 65 --
.../slider/common/params/ActionDestroyArgs.java | 37 -
.../common/params/ActionDiagnosticArgs.java | 4 +-
.../slider/common/params/ActionExistsArgs.java | 4 +-
.../slider/common/params/ActionFlexArgs.java | 55 --
.../slider/common/params/ActionFreezeArgs.java | 4 +-
.../slider/common/params/ActionHelpArgs.java | 4 +-
.../slider/common/params/ActionKDiagArgs.java | 4 +-
.../slider/common/params/ActionKeytabArgs.java | 4 +-
.../common/params/ActionKillContainerArgs.java | 4 +-
.../slider/common/params/ActionListArgs.java | 4 +-
.../slider/common/params/ActionLookupArgs.java | 4 +-
.../slider/common/params/ActionNodesArgs.java | 4 +-
.../common/params/ActionRegistryArgs.java | 9 +-
.../slider/common/params/ActionResolveArgs.java | 6 +-
.../common/params/ActionResourceArgs.java | 4 +-
.../slider/common/params/ActionStatusArgs.java | 4 +-
.../slider/common/params/ActionThawArgs.java | 4 +-
.../slider/common/params/ActionTokensArgs.java | 4 +-
.../slider/common/params/ActionUpdateArgs.java | 3 +-
.../slider/common/params/ActionUpgradeArgs.java | 5 +-
.../slider/common/params/ActionVersionArgs.java | 4 +-
.../org/apache/slider/common/params/ArgOps.java | 157 ----
.../apache/slider/common/params/Arguments.java | 138 ---
.../apache/slider/common/params/ClientArgs.java | 361 --------
.../apache/slider/common/params/CommonArgs.java | 300 -------
.../common/params/ComponentArgsDelegate.java | 52 --
.../slider/common/params/SliderAMArgs.java | 57 --
.../common/params/SliderAMCreateAction.java | 74 --
.../slider/common/params/SliderActions.java | 106 ---
.../slider/common/tools/ConfigHelper.java | 4 +-
.../slider/common/tools/CoreFileSystem.java | 12 +-
.../apache/slider/common/tools/PortScanner.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 18 +-
.../slider/core/exceptions/SliderException.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 253 +-----
.../slider/core/launch/ContainerLauncher.java | 26 +-
.../slider/core/launch/CredentialUtils.java | 2 +-
.../core/registry/SliderRegistryUtils.java | 2 +-
.../providers/AbstractClientProvider.java | 153 ----
.../providers/AbstractProviderService.java | 36 +-
.../slider/providers/DefaultClientProvider.java | 45 -
.../providers/DefaultProviderFactory.java | 47 --
.../providers/DefaultProviderService.java | 38 -
.../slider/providers/ProviderService.java | 68 --
.../apache/slider/providers/ProviderUtils.java | 525 ------------
.../slider/providers/SliderProviderFactory.java | 77 --
.../providers/docker/DockerClientProvider.java | 53 --
.../slider/providers/docker/DockerKeys.java | 30 -
.../providers/docker/DockerProviderFactory.java | 52 --
.../providers/docker/DockerProviderService.java | 66 --
.../tarball/TarballClientProvider.java | 65 --
.../tarball/TarballProviderFactory.java | 52 --
.../tarball/TarballProviderService.java | 54 --
.../server/appmaster/RoleLaunchService.java | 105 +--
.../server/appmaster/SliderAppMaster.java | 126 ++-
.../actions/ProviderReportedContainerLoss.java | 53 --
.../server/appmaster/metrics/SliderMetrics.java | 107 ---
.../appmaster/monkey/ChaosKillContainer.java | 2 +-
.../slider/server/appmaster/rpc/RpcBinder.java | 5 +-
.../appmaster/rpc/SliderAMPolicyProvider.java | 2 +-
.../appmaster/rpc/SliderRPCSecurityInfo.java | 2 +-
.../security/SecurityConfiguration.java | 4 +-
.../slider/server/appmaster/state/AppState.java | 62 +-
.../appmaster/state/AppStateBindingInfo.java | 3 -
.../server/appmaster/state/RoleHistory.java | 1 -
.../server/appmaster/state/RoleInstance.java | 1 -
.../server/appmaster/state/RoleStatus.java | 19 +-
.../ServiceTimelinePublisher.java | 372 ---------
.../timelineservice/SliderMetricsSink.java | 102 ---
.../SliderTimelineEntityType.java | 39 -
.../timelineservice/SliderTimelineEvent.java | 34 -
.../SliderTimelineMetricsConstants.java | 93 ---
.../appmaster/timelineservice/package-info.java | 27 -
.../server/appmaster/web/view/IndexBlock.java | 4 +-
.../slider/server/avro/RoleHistoryWriter.java | 2 +-
.../slider/server/servicemonitor/HttpProbe.java | 20 +-
.../slider/server/servicemonitor/PortProbe.java | 22 +-
.../slider/server/servicemonitor/Probe.java | 6 +-
.../YarnRegistryViewForProviders.java | 64 +-
.../apache/slider/util/RestApiConstants.java | 4 +-
.../org/apache/slider/util/ServiceApiUtil.java | 407 ---------
.../src/main/proto/ClientAMProtocol.proto | 56 ++
.../hadoop/yarn/service/TestServiceApiUtil.java | 539 ++++++++++++
.../yarn/service/TestYarnNativeServices.java | 496 +++++++++++
.../client/TestBuildExternalComponents.java | 128 +++
.../yarn/service/client/TestServiceCLI.java | 134 +++
.../yarn/service/conf/ExampleAppJson.java | 64 ++
.../yarn/service/conf/TestAppJsonResolve.java | 252 ++++++
.../service/conf/TestLoadExampleAppJson.java | 81 ++
.../service/conf/TestValidateServiceNames.java | 123 +++
.../providers/TestAbstractClientProvider.java | 119 +++
.../service/providers/TestProviderFactory.java | 75 ++
.../TestServiceTimelinePublisher.java | 311 +++++++
.../org/apache/slider/api/TestRPCBinding.java | 4 +-
.../apache/slider/client/TestClientBadArgs.java | 34 +-
.../slider/client/TestClientBasicArgs.java | 10 +-
.../slider/client/TestCommonArgParsing.java | 80 +-
.../slider/client/TestKeytabCommandOptions.java | 22 +-
.../slider/client/TestSliderClientMethods.java | 8 +-
.../slider/client/TestSliderTokensCommand.java | 18 +-
.../slider/common/tools/TestClusterNames.java | 122 ---
.../slider/common/tools/TestConfigHelper.java | 4 +-
.../common/tools/TestConfigHelperHDFS.java | 2 +-
.../common/tools/TestExecutionEnvironment.java | 14 +-
.../slider/common/tools/TestPortScan.java | 26 +-
.../common/tools/TestSliderFileSystem.java | 6 +-
.../common/tools/TestSliderTestUtils.java | 12 +-
.../slider/common/tools/TestSliderUtils.java | 10 +-
.../slider/common/tools/TestWindowsSupport.java | 24 +-
.../slider/common/tools/TestZKIntegration.java | 8 +-
.../apache/slider/core/conf/ExampleAppJson.java | 64 --
.../core/conf/TestConfigurationResolve.java | 252 ------
.../slider/core/conf/TestExampleAppJson.java | 81 --
.../TestPublishedConfigurationOutputter.java | 14 +-
.../slider/other/TestFilesystemPermissions.java | 12 +-
.../apache/slider/other/TestLocalDirStatus.java | 8 +-
.../providers/TestAbstractClientProvider.java | 3 +-
.../TestBuildApplicationComponent.java | 96 ---
.../slider/providers/TestDefaultProvider.java | 60 --
.../slider/providers/TestProviderFactory.java | 37 +-
.../slider/registry/TestConfigSetNaming.java | 10 +-
.../slider/registry/TestRegistryPaths.java | 8 +-
.../server/appmaster/actions/TestActions.java | 16 +-
.../TestMockAppStateAAOvercapacity.java | 2 +-
.../appstate/TestMockAppStateAAPlacement.java | 16 +-
.../TestMockAppStateContainerFailure.java | 30 +-
.../appstate/TestMockAppStateDependencies.java | 7 +-
.../TestMockAppStateDynamicHistory.java | 4 +-
.../appstate/TestMockAppStateDynamicRoles.java | 12 +-
.../TestMockAppStateFlexDynamicRoles.java | 6 +-
.../model/appstate/TestMockAppStateFlexing.java | 10 +-
.../appstate/TestMockAppStateRMOperations.java | 28 +-
.../TestMockAppStateRebuildOnAMRestart.java | 2 +-
.../appstate/TestMockAppStateRolePlacement.java | 2 +-
.../appstate/TestMockAppStateRoleRelease.java | 2 +-
.../appstate/TestMockAppStateUniqueNames.java | 4 +-
.../TestMockContainerResourceAllocations.java | 6 +-
.../appstate/TestMockLabelledAAPlacement.java | 4 +-
.../TestOutstandingRequestValidation.java | 8 +-
.../model/history/TestRoleHistoryAA.java | 26 +-
.../history/TestRoleHistoryContainerEvents.java | 18 +-
...TestRoleHistoryFindNodesForNewInstances.java | 14 +-
.../history/TestRoleHistoryNIComparators.java | 12 +-
...estRoleHistoryOutstandingRequestTracker.java | 26 +-
.../model/history/TestRoleHistoryRW.java | 22 +-
.../history/TestRoleHistoryRWOrdering.java | 12 +-
.../history/TestRoleHistoryRequestTracking.java | 20 +-
.../history/TestRoleHistoryUpdateBlacklist.java | 4 +-
.../model/mock/BaseMockAppStateTest.java | 2 +-
.../model/mock/MockProviderService.java | 149 ----
.../appmaster/model/monkey/TestMockMonkey.java | 20 +-
.../security/TestSecurityConfiguration.java | 22 +-
.../TestServiceTimelinePublisher.java | 304 -------
.../appmaster/timelineservice/package-info.java | 26 -
.../web/rest/registry/PathEntryMarshalling.java | 2 +-
.../registry/TestRegistryRestMarshalling.java | 2 +-
.../web/view/TestClusterSpecificationBlock.java | 2 +-
.../web/view/TestContainerStatsBlock.java | 14 +-
.../appmaster/web/view/TestIndexBlock.java | 2 +-
.../slider/server/management/TestGauges.java | 2 +-
.../server/servicemonitor/TestPortProbe.java | 42 -
.../workflow/TestWorkflowClosingService.java | 10 +-
.../workflow/TestWorkflowCompositeService.java | 14 +-
.../workflow/TestWorkflowExecutorService.java | 4 +-
.../workflow/TestWorkflowRpcService.java | 6 +-
.../workflow/TestWorkflowSequenceService.java | 18 +-
.../TestWorkflowServiceTerminatingRunnable.java | 6 +-
.../org/apache/slider/utils/KeysForTests.java | 4 +-
.../apache/slider/utils/SliderTestUtils.java | 4 +-
.../org/apache/slider/utils/TestAssertions.java | 6 +-
.../apache/slider/utils/TestServiceApiUtil.java | 523 ------------
.../slider/utils/YarnMiniClusterTestBase.java | 8 +-
.../src/test/resources/example-app.json | 15 +
.../service/conf/examples/app-override.json | 72 ++
.../hadoop/yarn/service/conf/examples/app.json | 47 ++
.../yarn/service/conf/examples/default.json | 16 +
.../yarn/service/conf/examples/external0.json | 8 +
.../yarn/service/conf/examples/external1.json | 30 +
.../yarn/service/conf/examples/external2.json | 22 +
.../yarn/service/provider/docker/appConfig.json | 42 +
.../yarn/service/provider/docker/resources.json | 16 +
.../yarn/service/provider/docker/test.template | 16 +
.../slider/core/conf/examples/app-override.json | 72 --
.../apache/slider/core/conf/examples/app.json | 47 --
.../slider/core/conf/examples/default.json | 16 -
.../slider/core/conf/examples/external0.json | 8 -
.../slider/core/conf/examples/external1.json | 30 -
.../slider/core/conf/examples/external2.json | 22 -
.../slider/providers/docker/appConfig.json | 42 -
.../slider/providers/docker/resources.json | 16 -
.../slider/providers/docker/test.template | 16 -
.../src/test/resources/yarn-site.xml | 19 +
.../hadoop/yarn/client/api/AMRMClient.java | 3 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 14 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 34 +
.../hadoop/yarn/util/BoundedAppender.java | 133 +++
.../hadoop/yarn/util/TestBoundedAppender.java | 115 +++
.../client/api/RegistryOperationsFactory.java | 8 +
.../registry/client/impl/zk/CuratorService.java | 5 +-
.../impl/zk/RegistryOperationsService.java | 4 +-
.../server/resourcemanager/RMAppManager.java | 56 +-
.../blacklist/SimpleBlacklistManager.java | 8 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 147 +---
.../scheduler/AbstractYarnScheduler.java | 11 +-
.../scheduler/SchedulerUtils.java | 17 +-
.../server/resourcemanager/TestRMRestart.java | 8 +-
.../rmapp/attempt/TestBoundedAppender.java | 116 ---
.../TestRMAppAttemptImplDiagnostics.java | 3 +-
297 files changed, 12492 insertions(+), 8197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 39ef8d2..98a2036 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,9 +47,9 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
hadoop_add_subcommand "rmadmin" "admin tools"
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
- hadoop_add_subcommand "servicesapi" "run slider services api"
+ hadoop_add_subcommand "servicesapi" "run yarn-service rest server"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
- hadoop_add_subcommand "slider" "run a slider app"
+ hadoop_add_subcommand "service" "run a service"
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "top" "view cluster information"
@@ -161,9 +161,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
- slider)
+ service)
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
- HADOOP_CLASSNAME='org.apache.slider.Slider'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 5a4de0e..decd849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -18,23 +18,21 @@
package org.apache.hadoop.yarn.services.api.impl;
import com.google.inject.Singleton;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ApplicationState;
import org.apache.slider.api.resource.ApplicationStatus;
import org.apache.slider.api.resource.Component;
-import org.apache.slider.util.ServiceApiUtil;
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.buildutils.BuildHelper;
-import org.apache.slider.core.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +49,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
import static org.apache.slider.util.RestApiConstants.*;
@@ -61,51 +58,39 @@ import static org.apache.slider.util.RestApiConstants.*;
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public class ApplicationApiService {
- private static final Logger logger =
+ private static final Logger LOG =
LoggerFactory.getLogger(ApplicationApiService.class);
- private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG =
- new YarnConfiguration();
- private static SliderClient SLIDER_CLIENT;
- private static Response SLIDER_VERSION;
- private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs();
+ private static Configuration YARN_CONFIG = new YarnConfiguration();
+ private static ServiceClient SERVICE_CLIENT;
static {
init();
}
// initialize all the common resources - order is important
- protected static void init() {
- SLIDER_CLIENT = createSliderClient();
- SLIDER_VERSION = initSliderVersion();
+ private static void init() {
+ SERVICE_CLIENT = new ServiceClient();
+ SERVICE_CLIENT.init(YARN_CONFIG);
+ SERVICE_CLIENT.start();
}
@GET
- @Path("/versions/slider-version")
+ @Path("/versions/yarn-service-version")
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() {
- logger.info("GET: getSliderVersion");
- return SLIDER_VERSION;
- }
-
- private static Response initSliderVersion() {
- Map<String, Object> metadata = new HashMap<>();
- BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services");
- String sliderVersion = metadata.toString();
- logger.info("Slider version = {}", sliderVersion);
- String hadoopVersion = SliderVersionInfo.getHadoopVersionString();
- logger.info("Hadoop version = {}", hadoopVersion);
- return Response.ok("{ \"slider_version\": \"" + sliderVersion
- + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
+ String version = VersionInfo.getBuildVersion();
+ LOG.info(version);
+ return Response.ok(version).build();
}
@POST @Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response createApplication(Application application) {
- logger.info("POST: createApplication = {}", application);
+ LOG.info("POST: createApplication = {}", application);
ApplicationStatus applicationStatus = new ApplicationStatus();
try {
- ApplicationId applicationId = SLIDER_CLIENT.actionCreate(application);
- logger.info("Successfully created application " + application.getName()
+ ApplicationId applicationId = SERVICE_CLIENT.actionCreate(application);
+ LOG.info("Successfully created application " + application.getName()
+ " applicationId = " + applicationId);
applicationStatus.setState(ApplicationState.ACCEPTED);
applicationStatus.setUri(
@@ -118,58 +103,18 @@ public class ApplicationApiService {
.build();
} catch (Exception e) {
String message = "Failed to create application " + application.getName();
- logger.error(message, e);
+ LOG.error(message, e);
applicationStatus.setDiagnostics(message + ": " + e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(applicationStatus).build();
}
}
- protected static SliderClient createSliderClient() {
- if (SLIDER_CLIENT != null) {
- return SLIDER_CLIENT;
- }
- org.apache.hadoop.conf.Configuration sliderClientConfiguration =
- SLIDER_CONFIG;
- SliderClient client = new SliderClient() {
- @Override public void init(org.apache.hadoop.conf.Configuration conf) {
- super.init(conf);
- try {
- initHadoopBinding();
- } catch (SliderException | IOException e) {
- throw new RuntimeException(
- "Unable to automatically init Hadoop binding", e);
- }
- }
- };
- try {
- logger
- .debug("Slider Client configuration: {}", sliderClientConfiguration);
- sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "help" });
- client.init(sliderClientConfiguration);
- client.start();
- } catch (Exception e) {
- logger.error("Unable to create SliderClient", e);
- throw new RuntimeException(e.getMessage(), e);
- }
- return client;
- }
-
- // The information this REST endpoint currently returned can be retrieved from
- // RM web services
- // Probably the data from AM is more important. Do that later.
-// @GET @Consumes({ MediaType.APPLICATION_JSON })
-// @Produces({ MediaType.APPLICATION_JSON })
-// public Response getApplications(@QueryParam("state") String state) {
-// logger.info("GET: getApplications with param state = {}", state);
-// return null;
-// }
-
@GET @Path("/{app_name}")
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response getApplication(@PathParam("app_name") String appName) {
- logger.info("GET: getApplication for appName = {}", appName);
+ LOG.info("GET: getApplication for appName = {}", appName);
ApplicationStatus applicationStatus = new ApplicationStatus();
// app name validation
@@ -181,24 +126,25 @@ public class ApplicationApiService {
}
try {
- Application app = SLIDER_CLIENT.actionStatus(appName);
- ApplicationReport report = SLIDER_CLIENT.findInstance(appName);
- if (app != null && report != null) {
+ Application app = SERVICE_CLIENT.getStatus(appName);
+ ApplicationReport report = SERVICE_CLIENT.getYarnClient()
+ .getApplicationReport(ApplicationId.fromString(app.getId()));
+ if (report != null) {
app.setLifetime(
report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME)
.getRemainingTime());
- logger.info("Application = {}", app);
+ LOG.info("Application = {}", app);
return Response.ok(app).build();
} else {
String message = "Application " + appName + " does not exist.";
- logger.info(message);
+ LOG.info(message);
applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST);
applicationStatus.setDiagnostics(message);
return Response.status(Status.NOT_FOUND).entity(applicationStatus)
.build();
}
} catch (Exception e) {
- logger.error("Get application failed", e);
+ LOG.error("Get application failed", e);
applicationStatus
.setDiagnostics("Failed to retrieve application: " + e.getMessage());
return Response.status(Status.INTERNAL_SERVER_ERROR)
@@ -211,18 +157,18 @@ public class ApplicationApiService {
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response deleteApplication(@PathParam("app_name") String appName) {
- logger.info("DELETE: deleteApplication for appName = {}", appName);
+ LOG.info("DELETE: deleteApplication for appName = {}", appName);
return stopApplication(appName, true);
}
private Response stopApplication(String appName, boolean destroy) {
try {
- SLIDER_CLIENT.actionStop(appName, ACTION_FREEZE_ARGS);
+ SERVICE_CLIENT.actionStop(appName);
if (destroy) {
- SLIDER_CLIENT.actionDestroy(appName);
- logger.info("Successfully deleted application {}", appName);
+ SERVICE_CLIENT.actionDestroy(appName);
+ LOG.info("Successfully deleted application {}", appName);
} else {
- logger.info("Successfully stopped application {}", appName);
+ LOG.info("Successfully stopped application {}", appName);
}
return Response.status(Status.NO_CONTENT).build();
} catch (ApplicationNotFoundException e) {
@@ -252,8 +198,8 @@ public class ApplicationApiService {
.getNumberOfContainers()).build();
}
try {
- Map<String, Long> original = SLIDER_CLIENT.flex(appName, Collections
- .singletonMap(component.getName(),
+ Map<String, Long> original = SERVICE_CLIENT.flexByRestService(appName,
+ Collections.singletonMap(component.getName(),
component.getNumberOfContainers()));
return Response.ok().entity("Updating " + componentName + " size from "
+ original.get(componentName) + " to "
@@ -271,7 +217,7 @@ public class ApplicationApiService {
@Produces({ MediaType.APPLICATION_JSON })
public Response updateApplication(@PathParam("app_name") String appName,
Application updateAppData) {
- logger.info("PUT: updateApplication for app = {} with data = {}", appName,
+ LOG.info("PUT: updateApplication for app = {} with data = {}", appName,
updateAppData);
// Ignore the app name provided in updateAppData and always use appName
@@ -314,14 +260,14 @@ public class ApplicationApiService {
private Response updateLifetime(String appName, Application updateAppData) {
try {
String newLifeTime =
- SLIDER_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
+ SERVICE_CLIENT.updateLifetime(appName, updateAppData.getLifetime());
return Response.ok("Application " + appName + " lifeTime is successfully updated to "
+ updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build();
} catch (Exception e) {
String message =
"Failed to update application (" + appName + ") lifetime ("
+ updateAppData.getLifetime() + ")";
- logger.error(message, e);
+ LOG.error(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message + " : " + e.getMessage()).build();
}
@@ -329,17 +275,12 @@ public class ApplicationApiService {
private Response startApplication(String appName) {
try {
- int ret = SLIDER_CLIENT.actionList(appName);
- if (ret == 0) {
- return Response.ok()
- .entity("Application " + appName + " is already alive.").build();
- }
- SLIDER_CLIENT.actionStart(appName, null);
- logger.info("Successfully started application " + appName);
+ SERVICE_CLIENT.actionStart(appName);
+ LOG.info("Successfully started application " + appName);
return Response.ok("Application " + appName + " is successfully started").build();
} catch (Exception e) {
String message = "Failed to start application " + appName;
- logger.info(message, e);
+ LOG.info(message, e);
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(message + ": " + e.getMessage()).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
index e5cde4e..f2bf582 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -17,6 +17,14 @@
-->
<FindBugsFilter>
<Match>
+ <Package name="org.apache.hadoop.yarn.proto" />
+ </Match>
+ <Match>
+ <class name="org.apache.hadoop.yarn.service.utils.ServiceApiUtil" />
+ <Bug pattern="MS_CANNOT_BE_FINAL" />
+ </Match>
+
+ <Match>
<Package name="org.apache.slider.api.proto" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index a1d1937..02317e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -61,6 +61,7 @@
<includes>
<include>SliderClusterMessages.proto</include>
<include>SliderClusterProtocol.proto</include>
+ <include>ClientAMProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
new file mode 100644
index 0000000..516d23d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+
+import java.io.IOException;
+
+public interface ClientAMProtocol {
+ FlexComponentsResponseProto flexComponents(FlexComponentsRequestProto request)
+ throws IOException, YarnException;
+
+ GetStatusResponseProto getStatus(GetStatusRequestProto requestProto)
+ throws IOException, YarnException;
+
+ StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
new file mode 100644
index 0000000..6884757
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+
+public class ClientAMService extends AbstractService
+ implements ClientAMProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ClientAMService.class);
+
+ private ServiceContext context;
+ private Server server;
+
+ private InetSocketAddress bindAddress;
+
+ public ClientAMService(ServiceContext context) {
+ super("Client AM Service");
+ this.context = context;
+ }
+
+ @Override protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress address = new InetSocketAddress(0);
+ server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
+ context.secretManager, 1);
+ server.start();
+
+ String nodeHostString =
+ System.getenv(ApplicationConstants.Environment.NM_HOST.name());
+
+ bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
+ server.getListenerAddress().getPort());
+
+ LOG.info("Instantiated ClientAMService at " + bindAddress);
+ super.serviceStart();
+ }
+
+ @Override protected void serviceStop() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override public FlexComponentsResponseProto flexComponents(
+ FlexComponentsRequestProto request) throws IOException {
+ if (!request.getComponentsList().isEmpty()) {
+ for (ComponentCountProto component : request.getComponentsList()) {
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getNumberOfContainers());
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ LOG.info("Flexing component {} to {}", component.getName(),
+ component.getNumberOfContainers());
+ }
+ }
+ return FlexComponentsResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public GetStatusResponseProto getStatus(GetStatusRequestProto request)
+ throws IOException, YarnException {
+ String stat = ServiceApiUtil.jsonSerDeser.toJson(context.application);
+ return GetStatusResponseProto.newBuilder().setStatus(stat).build();
+ }
+
+ @Override
+ public StopResponseProto stop(StopRequestProto requestProto)
+ throws IOException, YarnException {
+ LOG.info("Stop the service.");
+ // Stop the service in 2 seconds delay to make sure this rpc call is completed.
+ // shutdown hook will be executed which will stop AM gracefully.
+ Thread thread = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(2000);
+ ExitUtil.terminate(0);
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted while stopping", e);
+ }
+ }
+ };
+ thread.start();
+ return StopResponseProto.newBuilder().build();
+ }
+
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
new file mode 100644
index 0000000..bbb4c44
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerFailureTracker.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.component.Component;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD;
+
+/**
+ * This tracks the container failures per node. If the failure counter exceeds
+ * the maxFailurePerNode limit, it'll blacklist that node.
+ *
+ */
+public class ContainerFailureTracker {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerFailureTracker.class);
+
+ // Host -> num container failures
+ private Map<String, Integer> failureCountPerNode = new HashMap<>();
+ private Set<String> blackListedNodes = new HashSet<>();
+ private ServiceContext context;
+ private int maxFailurePerNode;
+ private Component component;
+
+ public ContainerFailureTracker(ServiceContext context, Component component) {
+ this.context = context;
+ this.component = component;
+ maxFailurePerNode = component.getComponentSpec().getConfiguration()
+ .getPropertyInt(NODE_FAILURE_THRESHOLD, 3);
+ }
+
+
+ public synchronized void incNodeFailure(String host) {
+ int num = 0;
+ if (failureCountPerNode.containsKey(host)) {
+ num = failureCountPerNode.get(host);
+ }
+ num++;
+ failureCountPerNode.put(host, num);
+
+ // black list the node if exceed max failure
+ if (num > maxFailurePerNode && !blackListedNodes.contains(host)) {
+ List<String> blacklists = new ArrayList<>();
+ blacklists.add(host);
+ blackListedNodes.add(host);
+ context.scheduler.getAmRMClient().updateBlacklist(blacklists, null);
+ LOG.info("[COMPONENT {}]: Failed {} times on this host, blacklisted {}."
+ + " Current list of blacklisted nodes: {}",
+ component.getName(), num, host, blackListedNodes);
+ }
+ }
+
+ public synchronized void resetContainerFailures() {
+ // reset container failure counter per node
+ failureCountPerNode.clear();
+ context.scheduler.getAmRMClient()
+ .updateBlacklist(null, new ArrayList<>(blackListedNodes));
+ LOG.info("[COMPONENT {}]: Clearing blacklisted nodes {} ",
+ component.getName(), blackListedNodes);
+ blackListedNodes.clear();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
new file mode 100644
index 0000000..ac5285f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.launch.AbstractLauncher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ContainerLaunchService extends AbstractService{
+
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLaunchService.class);
+
+ private ExecutorService executorService;
+ private SliderFileSystem fs;
+
+ public ContainerLaunchService(SliderFileSystem fs) {
+ super(ContainerLaunchService.class.getName());
+ this.fs = fs;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newCachedThreadPool();
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ super.serviceStop();
+ }
+
+ public void launchCompInstance(Application application,
+ ComponentInstance instance, Container container) {
+ ContainerLauncher launcher =
+ new ContainerLauncher(application, instance, container);
+ executorService.execute(launcher);
+ }
+
+ private class ContainerLauncher implements Runnable {
+ public final Container container;
+ public final Application application;
+ public ComponentInstance instance;
+
+ public ContainerLauncher(
+ Application application,
+ ComponentInstance instance, Container container) {
+ this.container = container;
+ this.application = application;
+ this.instance = instance;
+ }
+
+ @Override public void run() {
+ org.apache.slider.api.resource.Component compSpec = instance.getCompSpec();
+ ProviderService provider = ProviderFactory.getProviderService(
+ compSpec.getArtifact());
+ AbstractLauncher launcher = new AbstractLauncher(fs, null);
+ try {
+ provider.buildContainerLaunchContext(launcher, application,
+ instance, fs);
+ instance.getComponent().getScheduler().getNmClient()
+ .startContainerAsync(container,
+ launcher.completeContainerLaunch());
+ } catch (Exception e) {
+ LOG.error(instance.getCompInstanceId()
+ + ": Failed to launch container. ", e);
+
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
new file mode 100644
index 0000000..80668a0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceContext.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.LoadingCache;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+public class ServiceContext {
+ public Application application = null;
+ public SliderFileSystem fs;
+ public String serviceHdfsDir = "";
+ public ApplicationAttemptId attemptId;
+ public LoadingCache<ConfigFile, Object> configCache;
+ public ServiceScheduler scheduler;
+ public ClientToAMTokenSecretManager secretManager;
+ public ClientAMService clientAMService;
+
+ public ServiceContext() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
new file mode 100644
index 0000000..1ebd562
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.service.client.params.SliderAMArgs;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class ServiceMaster extends CompositeService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMaster.class);
+
+ private static SliderAMArgs amArgs;
+
+ public ServiceMaster(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ //TODO Deprecate slider conf, make sure works with yarn conf
+ printSystemEnv();
+ if (UserGroupInformation.isSecurityEnabled()) {
+ UserGroupInformation.setConfiguration(conf);
+ }
+ LOG.info("Login user is {}", UserGroupInformation.getLoginUser());
+
+ ServiceContext context = new ServiceContext();
+ Path appDir = new Path(amArgs.getAppDefPath()).getParent();
+ SliderFileSystem fs = new SliderFileSystem(conf);
+ context.fs = fs;
+ fs.setAppDir(appDir);
+ context.application = ServiceApiUtil
+ .loadApplicationFrom(fs, new Path(amArgs.getAppDefPath()));
+ LOG.info(context.application.toString());
+ ContainerId amContainerId = ContainerId.fromString(SliderUtils
+ .mandatoryEnvVariable(
+ ApplicationConstants.Environment.CONTAINER_ID.name()));
+ ApplicationAttemptId attemptId = amContainerId.getApplicationAttemptId();
+ LOG.info("Application attemptId: " + attemptId);
+ context.attemptId = attemptId;
+
+ // configure AM to wait forever for RM
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+ conf.unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
+ DefaultMetricsSystem.initialize("ServiceAppMaster");
+
+ context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
+ ClientAMService clientAMService = new ClientAMService(context);
+ context.clientAMService = clientAMService;
+ addService(clientAMService);
+
+ ServiceScheduler scheduler = new ServiceScheduler(context);
+ addService(scheduler);
+ context.scheduler = scheduler;
+
+ ServiceMonitor monitor = new ServiceMonitor("Service Monitor", context);
+ addService(monitor);
+
+ super.serviceInit(conf);
+ }
+
+
+ @Override
+ protected void serviceStop() throws Exception {
+ LOG.info("Stopping app master");
+ super.serviceStop();
+ }
+
+ private void printSystemEnv() {
+ for (Map.Entry<String, String> envs : System.getenv().entrySet()) {
+ LOG.info("{} = {}", envs.getKey(), envs.getValue());
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
+ amArgs = new SliderAMArgs(args);
+ amArgs.parse();
+ try {
+ ServiceMaster serviceMaster = new ServiceMaster("Service Master");
+ ShutdownHookManager.get()
+ .addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
+ YarnConfiguration conf = new YarnConfiguration();
+ new GenericOptionsParser(conf, args);
+ serviceMaster.init(conf);
+ serviceMaster.start();
+ } catch (Throwable t) {
+ LOG.error("Error starting service master", t);
+ ExitUtil.terminate(1, "Error starting service master");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
new file mode 100644
index 0000000..82d768e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMonitor.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.slider.api.InternalKeys;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.servicemonitor.ProbeStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_NOT_READY;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.BECOME_READY;
+import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.READY;
+
+public class ServiceMonitor extends AbstractService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ServiceMonitor.class);
+
+ public ScheduledExecutorService executorService;
+ private Map<ContainerId, ComponentInstance> liveInstances = null;
+ private ServiceContext context;
+
+ public ServiceMonitor(String name, ServiceContext context) {
+ super(name);
+ liveInstances = context.scheduler.getLiveInstances();
+ this.context = context;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ executorService = Executors.newScheduledThreadPool(1);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ long readinessCheckInterval = context.application.getConfiguration()
+ .getPropertyLong(InternalKeys.MONITOR_INTERVAL,
+ InternalKeys.DEFAULT_MONITOR_INTERVAL);
+ executorService
+ .scheduleAtFixedRate(new ReadinessChecker(), readinessCheckInterval,
+ readinessCheckInterval, TimeUnit.SECONDS);
+
+ long failureResetInterval = SliderUtils
+ .getTimeRange(context.application.getConfiguration(),
+ ResourceKeys.CONTAINER_FAILURE_WINDOW,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0);
+
+ executorService
+ .scheduleAtFixedRate(new ContainerFailureReset(), failureResetInterval,
+ failureResetInterval, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void serviceStop() throws Exception {
+ if (executorService != null) {
+ executorService.shutdownNow();
+ }
+ }
+
+ private class ReadinessChecker implements Runnable {
+
+ @Override
+ public void run() {
+
+ // check if the comp instance are ready
+ for (Map.Entry<ContainerId, ComponentInstance> entry : liveInstances
+ .entrySet()) {
+ ComponentInstance instance = entry.getValue();
+
+ ProbeStatus status = instance.ping();
+ if (status.isSuccess()) {
+ if (instance.getState() != READY) {
+ // synchronously update the state.
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_READY));
+ }
+ } else {
+ if (instance.getState() == READY) {
+ instance.handle(
+ new ComponentInstanceEvent(entry.getKey(), BECOME_NOT_READY));
+ }
+ }
+ }
+
+ for (Component component : context.scheduler.getAllComponents()
+ .values()) {
+ // If comp hasn't started yet and its dependencies are satisfied
+ if (component.getState() == ComponentState.INIT && component
+ .areDependenciesReady()) {
+ LOG.info("[COMPONENT {}]: Dependencies satisfied, ramping up.",
+ component.getName());
+ ComponentEvent event = new ComponentEvent(component.getName(), FLEX)
+ .setDesired(component.getComponentSpec().getNumberOfContainers());
+ component.handle(event);
+ }
+ }
+ }
+ }
+
+ private class ContainerFailureReset implements Runnable {
+ @Override
+ public void run() {
+ for (Component component : context.scheduler.getAllComponents().values()) {
+ component.resetCompFailureCount();
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org