You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:14 UTC
[32/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/256a1597
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/256a1597
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/256a1597
Branch: refs/heads/yarn-native-services
Commit: 256a1597671a9cd872b8204b8c81ce2c9a8f1cea
Parents: e2e4559
Author: Jian He <ji...@apache.org>
Authored: Thu Apr 20 23:53:04 2017 +0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Jul 25 10:49:25 2017 -0700
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 11 +-
.../hadoop-yarn-slider-core/pom.xml | 31 +-
.../org/apache/slider/api/InternalKeys.java | 6 +-
.../org/apache/slider/api/ResourceKeys.java | 15 +-
.../slider/api/SliderClusterProtocol.java | 4 +-
.../slider/api/resource/Configuration.java | 22 +
.../apache/slider/api/types/RoleStatistics.java | 6 -
.../org/apache/slider/client/SliderClient.java | 512 ++-------
.../apache/slider/client/SliderClientAPI.java | 62 +-
.../client/ipc/SliderClusterOperations.java | 20 +-
.../apache/slider/common/SliderXmlConfKeys.java | 19 -
.../AbstractClusterBuildingActionArgs.java | 53 +
.../slider/common/params/ActionBuildArgs.java | 31 +
.../slider/common/params/ActionCreateArgs.java | 12 -
.../slider/common/params/ActionEchoArgs.java | 33 -
.../slider/common/params/ActionFlexArgs.java | 33 +-
.../common/params/ActionInstallKeytabArgs.java | 57 -
.../common/params/ActionInstallPackageArgs.java | 58 -
.../slider/common/params/ActionKeytabArgs.java | 7 -
.../slider/common/params/ActionPackageArgs.java | 81 --
.../slider/common/params/ActionUpgradeArgs.java | 2 +-
.../params/AppAndResouceOptionArgsDelegate.java | 111 --
.../apache/slider/common/params/Arguments.java | 26 -
.../apache/slider/common/params/ClientArgs.java | 34 +-
.../common/params/ComponentArgsDelegate.java | 2 +-
.../common/params/OptionArgsDelegate.java | 66 ++
.../slider/common/params/SliderActions.java | 7 -
.../slider/common/tools/ConfigHelper.java | 8 -
.../apache/slider/common/tools/SliderUtils.java | 118 +-
.../slider/core/launch/AbstractLauncher.java | 1 -
.../apache/slider/core/zk/ZKIntegration.java | 9 +-
.../apache/slider/providers/ProviderRole.java | 12 +-
.../server/appmaster/SliderAppMaster.java | 63 +-
.../appmaster/actions/ActionFlexCluster.java | 4 +-
.../server/appmaster/metrics/SliderMetrics.java | 12 +-
.../rpc/SliderClusterProtocolPBImpl.java | 6 +-
.../rpc/SliderClusterProtocolProxy.java | 6 +-
.../server/appmaster/rpc/SliderIPCService.java | 6 +-
.../security/SecurityConfiguration.java | 235 ++--
.../slider/server/appmaster/state/AppState.java | 257 +++--
.../appmaster/state/ContainerOutcome.java | 4 +-
.../server/appmaster/state/NodeEntry.java | 2 +-
.../server/appmaster/state/RoleStatus.java | 46 +-
.../server/appmaster/web/view/IndexBlock.java | 4 +-
.../src/main/proto/SliderClusterMessages.proto | 10 +-
.../src/main/proto/SliderClusterProtocol.proto | 2 +-
.../org/apache/slider/api/TestRPCBinding.java | 50 +
.../apache/slider/client/TestClientBadArgs.java | 229 ++++
.../slider/client/TestClientBasicArgs.java | 81 ++
.../slider/client/TestCommonArgParsing.java | 522 +++++++++
.../slider/client/TestKeytabCommandOptions.java | 405 +++++++
.../slider/client/TestSliderClientMethods.java | 142 +++
.../slider/client/TestSliderTokensCommand.java | 124 ++
.../slider/common/tools/TestClusterNames.java | 122 ++
.../slider/common/tools/TestConfigHelper.java | 57 +
.../common/tools/TestConfigHelperHDFS.java | 57 +
.../common/tools/TestExecutionEnvironment.java | 67 ++
.../common/tools/TestMiscSliderUtils.java | 49 +
.../slider/common/tools/TestPortScan.java | 184 +++
.../common/tools/TestSliderFileSystem.java | 62 +
.../common/tools/TestSliderTestUtils.java | 97 ++
.../slider/common/tools/TestSliderUtils.java | 25 -
.../slider/common/tools/TestWindowsSupport.java | 177 +++
.../slider/common/tools/TestZKIntegration.java | 187 +++
.../slider/core/conf/ExampleConfResources.java | 58 +
.../core/conf/TestConfTreeLoadExamples.java | 64 ++
.../core/conf/TestConfigurationResolve.java | 118 ++
.../slider/other/TestFilesystemPermissions.java | 263 +++++
.../apache/slider/other/TestLocalDirStatus.java | 166 +++
.../slider/providers/TestProviderFactory.java | 54 +
.../slider/registry/TestConfigSetNaming.java | 85 ++
.../slider/registry/TestRegistryPaths.java | 74 ++
.../server/appmaster/actions/TestActions.java | 246 ++++
.../model/appstate/BaseMockAppStateAATest.java | 73 ++
.../TestMockAppStateAAOvercapacity.java | 112 ++
.../appstate/TestMockAppStateAAPlacement.java | 380 +++++++
.../TestMockAppStateContainerFailure.java | 387 +++++++
.../TestMockAppStateDynamicHistory.java | 212 ++++
.../appstate/TestMockAppStateDynamicRoles.java | 243 ++++
.../TestMockAppStateFlexDynamicRoles.java | 160 +++
.../model/appstate/TestMockAppStateFlexing.java | 201 ++++
.../appstate/TestMockAppStateRMOperations.java | 382 +++++++
.../TestMockAppStateRebuildOnAMRestart.java | 117 ++
.../appstate/TestMockAppStateRolePlacement.java | 122 ++
.../appstate/TestMockAppStateRoleRelease.java | 82 ++
.../appstate/TestMockAppStateUniqueNames.java | 111 ++
.../TestMockContainerResourceAllocations.java | 89 ++
.../appstate/TestMockLabelledAAPlacement.java | 156 +++
.../TestOutstandingRequestValidation.java | 110 ++
.../model/history/TestRoleHistoryAA.java | 269 +++++
.../history/TestRoleHistoryContainerEvents.java | 447 ++++++++
...TestRoleHistoryFindNodesForNewInstances.java | 177 +++
.../history/TestRoleHistoryNIComparators.java | 133 +++
...estRoleHistoryOutstandingRequestTracker.java | 385 +++++++
.../model/history/TestRoleHistoryRW.java | 371 ++++++
.../history/TestRoleHistoryRWOrdering.java | 162 +++
.../history/TestRoleHistoryRequestTracking.java | 298 +++++
.../history/TestRoleHistoryUpdateBlacklist.java | 117 ++
.../server/appmaster/model/mock/Allocator.java | 123 ++
.../model/mock/BaseMockAppStateTest.java | 524 +++++++++
.../server/appmaster/model/mock/MockAM.java | 26 +
.../appmaster/model/mock/MockAppState.java | 82 ++
.../model/mock/MockApplicationAttemptId.java | 61 +
.../appmaster/model/mock/MockApplicationId.java | 67 ++
.../model/mock/MockClusterServices.java | 38 +
.../appmaster/model/mock/MockContainer.java | 131 +++
.../appmaster/model/mock/MockContainerId.java | 104 ++
.../appmaster/model/mock/MockFactory.java | 270 +++++
.../appmaster/model/mock/MockFileSystem.java | 32 +
.../server/appmaster/model/mock/MockNodeId.java | 62 +
.../appmaster/model/mock/MockPriority.java | 46 +
.../model/mock/MockProviderService.java | 140 +++
.../model/mock/MockRMOperationHandler.java | 120 ++
.../appmaster/model/mock/MockRecordFactory.java | 27 +
.../model/mock/MockRegistryOperations.java | 83 ++
.../appmaster/model/mock/MockResource.java | 75 ++
.../appmaster/model/mock/MockRoleHistory.java | 53 +
.../server/appmaster/model/mock/MockRoles.java | 30 +
.../appmaster/model/mock/MockYarnCluster.java | 342 ++++++
.../appmaster/model/mock/MockYarnEngine.java | 188 ++++
.../appmaster/model/monkey/TestMockMonkey.java | 208 ++++
.../security/TestSecurityConfiguration.java | 215 ++++
.../TestServiceTimelinePublisher.java | 24 +-
.../web/rest/registry/PathEntryMarshalling.java | 28 +
.../registry/TestRegistryRestMarshalling.java | 51 +
.../web/view/TestClusterSpecificationBlock.java | 78 ++
.../web/view/TestContainerStatsBlock.java | 255 +++++
.../appmaster/web/view/TestIndexBlock.java | 175 +++
.../slider/server/management/TestGauges.java | 55 +
.../server/servicemonitor/TestPortProbe.java | 2 +-
.../apache/slider/test/ContractTestUtils.java | 901 ---------------
.../org/apache/slider/tools/TestUtility.java | 181 ---
.../apache/slider/utils/ContractTestUtils.java | 901 +++++++++++++++
.../org/apache/slider/utils/KeysForTests.java | 38 +
.../org/apache/slider/utils/MicroZKCluster.java | 87 ++
.../java/org/apache/slider/utils/Outcome.java | 46 +
.../org/apache/slider/utils/SliderTestBase.java | 60 +
.../apache/slider/utils/SliderTestUtils.java | 1065 ++++++++++++++++++
.../org/apache/slider/utils/TestAssertions.java | 60 +
.../org/apache/slider/utils/TestUtility.java | 181 +++
.../slider/utils/YarnMiniClusterTestBase.java | 832 ++++++++++++++
.../slider/utils/YarnZKMiniClusterTestBase.java | 179 +++
.../src/test/resources/log4j.properties | 66 ++
.../slider/common/tools/test/metainfo.txt | 16 -
.../slider/common/tools/test/metainfo.xml | 98 --
.../slider/common/tools/test/someOtherFile.txt | 16 -
.../slider/common/tools/test/someOtherFile.xml | 17 -
.../conf/examples/app-override-resolved.json | 49 +
.../slider/core/conf/examples/app-override.json | 43 +
.../slider/core/conf/examples/app-resolved.json | 81 ++
.../apache/slider/core/conf/examples/app.json | 54 +
.../agent/application/metadata/metainfo.xml | 180 ---
.../appmaster/web/rest/registry/sample.json | 9 +
.../slider/server/avro/history-v01-3-role.json | 6 +
.../slider/server/avro/history-v01-6-role.json | 8 +
.../slider/server/avro/history_v01b_1_role.json | 38 +
156 files changed, 17407 insertions(+), 2707 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index b4f6a2e..5a4de0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,6 +50,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -251,10 +252,12 @@ public class ApplicationApiService {
.getNumberOfContainers()).build();
}
try {
- long original = SLIDER_CLIENT.flex(appName, component);
- return Response.ok().entity(
- "Updating " + componentName + " size from " + original + " to "
- + component.getNumberOfContainers()).build();
+ Map<String, Long> original = SLIDER_CLIENT.flex(appName, Collections
+ .singletonMap(component.getName(),
+ component.getNumberOfContainers()));
+ return Response.ok().entity("Updating " + componentName + " size from "
+ + original.get(componentName) + " to "
+ + component.getNumberOfContainers()).build();
} catch (YarnException | IOException e) {
ApplicationStatus status = new ApplicationStatus();
status.setDiagnostics(e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 16a2bb2..05c4d88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -327,16 +327,38 @@
<artifactId>easymock</artifactId>
<version>3.1</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-easymock</artifactId>
- <version>1.5</version>
+ <version>1.6.5</version>
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>1.6.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.objenesis</groupId>
+ <artifactId>objenesis</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<scope>runtime</scope>
@@ -359,6 +381,13 @@
<artifactId>swagger-annotations</artifactId>
<version>1.5.4</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index fcaaf0e..f690f5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -126,9 +126,9 @@ public interface InternalKeys {
String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
- int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
- int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
- int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+ long DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
+ long DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
+ long DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 92890be..2f71004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -72,6 +72,15 @@ public interface ResourceKeys {
String YARN_CORES = "yarn.vcores";
/**
+ * If normalization is set to false, then if the resource (memory and/or
+ * vcore) requested by a role is higher than YARN limits, then the resource
+ * request is not normalized. If this causes failures at the YARN level then
+ * applications are expecting that to happen. Default value is true.
+ */
+ String YARN_RESOURCE_NORMALIZATION_ENABLED =
+ "yarn.resource.normalization.enabled";
+
+ /**
* Number of disks per instance to ask YARN for
* {@value}
*/
@@ -140,9 +149,9 @@ public interface ResourceKeys {
- int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
- int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
- int DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0;
+ long DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
+ long DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
+ long DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES = 0;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 7f768b9..448d4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -53,8 +53,8 @@ public interface SliderClusterProtocol extends VersionedProtocol {
YarnException;
- Messages.FlexComponentResponseProto flexComponent(
- Messages.FlexComponentRequestProto request) throws IOException;
+ Messages.FlexComponentsResponseProto flexComponents(
+ Messages.FlexComponentsRequestProto request) throws IOException;
/**
* Get the current cluster status
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
index c43bd64..7b3b93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
@@ -116,6 +116,28 @@ public class Configuration implements Serializable {
return Long.parseLong(value);
}
+ public int getPropertyInt(String name, int defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ String value = properties.get(name.trim());
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ public boolean getPropertyBool(String name, boolean defaultValue) {
+ if (name == null) {
+ return defaultValue;
+ }
+ String value = properties.get(name.trim());
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
public String getProperty(String name, String defaultValue) {
if (name == null) {
return defaultValue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
index c926600..25f4d9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
@@ -35,11 +35,8 @@ public class RoleStatistics {
public long limitsExceeded = 0L;
public long nodeFailed = 0L;
public long preempted = 0L;
- public long releasing = 0L;
public long requested = 0L;
public long started = 0L;
- public long startFailed = 0L;
- public long totalRequested = 0L;
/**
* Add another statistics instance
@@ -56,11 +53,8 @@ public class RoleStatistics {
limitsExceeded += that.limitsExceeded;
nodeFailed += that.nodeFailed;
preempted += that.preempted;
- releasing += that.releasing;
requested += that.requested;
started += that.started;
- startFailed += that.totalRequested;
- totalRequested += that.totalRequested;
return this;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8bceddf..2b0982f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -20,12 +20,9 @@ package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.Files;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
@@ -67,7 +64,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.resource.Application;
@@ -83,22 +79,17 @@ import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionDependencyArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionExistsArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionInstallKeytabArgs;
-import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKDiagArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionLookupArgs;
import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionResourceArgs;
@@ -122,7 +113,6 @@ import org.apache.slider.core.exceptions.NotFoundException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.launch.CredentialUtils;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
@@ -144,7 +134,6 @@ import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.SliderProviderFactory;
-import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
@@ -160,11 +149,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
-import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
@@ -177,8 +164,6 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -226,7 +211,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private ClientArgs serviceArgs;
public ApplicationId applicationId;
-
+
private String deployedClusterName;
/**
* Cluster operations against the deployed cluster -will be null
@@ -334,23 +319,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
exitCode = actionAmSuicide(clusterName,
serviceArgs.getActionAMSuicideArgs());
break;
-
+
+ case ACTION_BUILD:
+ exitCode = actionBuild(getApplicationFromArgs(clusterName,
+ serviceArgs.getActionBuildArgs()));
+ break;
+
case ACTION_CLIENT:
exitCode = actionClient(serviceArgs.getActionClientArgs());
break;
case ACTION_CREATE:
- ActionCreateArgs args = serviceArgs.getActionCreateArgs();
- File file = args.getAppDef();
- Path filePath = new Path(file.getAbsolutePath());
- log.info("Loading app definition from: " + filePath);
- Application application =
- jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath);
- if(args.lifetime > 0) {
- application.setLifetime(args.lifetime);
- }
- application.setName(clusterName);
- actionCreate(application);
+ actionCreate(getApplicationFromArgs(clusterName,
+ serviceArgs.getActionCreateArgs()));
break;
case ACTION_DEPENDENCY:
@@ -391,14 +372,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionKillContainerArgs());
break;
- case ACTION_INSTALL_KEYTAB:
- exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
- break;
-
- case ACTION_INSTALL_PACKAGE:
- exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
- break;
-
case ACTION_KEYTAB:
exitCode = actionKeytab(serviceArgs.getActionKeytabArgs());
break;
@@ -415,10 +388,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
exitCode = actionNodes("", serviceArgs.getActionNodesArgs());
break;
- case ACTION_PACKAGE:
- exitCode = actionPackage(serviceArgs.getActionPackageArgs());
- break;
-
case ACTION_REGISTRY:
exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
break;
@@ -605,15 +574,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* force=true by default.
*/
@Override
- public void actionDestroy(String appName)
+ public int actionDestroy(String appName)
throws YarnException, IOException {
validateClusterName(appName);
+ verifyNoLiveApp(appName, "Destroy");
Path appDir = sliderFileSystem.buildClusterDirPath(appName);
FileSystem fs = sliderFileSystem.getFileSystem();
if (fs.exists(appDir)) {
if (fs.delete(appDir, true)) {
- log.info("Successfully deleted application + " + appName);
- return;
+ log.info("Successfully deleted application dir for " + appName);
} else {
String message =
"Failed to delete application + " + appName + " at: " + appDir;
@@ -627,7 +596,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.warn(message);
throw new YarnException(message);
}
- //TODO clean registry
+
+ //TODO clean registry?
+ String registryPath = SliderRegistryUtils.registryPathForInstance(
+ appName);
+ try {
+ getRegistryOperations().delete(registryPath, true);
+ } catch (IOException e) {
+ log.warn("Error deleting registry entry {}: {} ", registryPath, e, e);
+ } catch (SliderException e) {
+ log.warn("Error binding to registry {} ", e, e);
+ }
+
+ log.info("Destroyed cluster {}", appName);
+ return EXIT_SUCCESS;
}
@@ -648,6 +630,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return factory.createClientProvider();
}
+ private Application getApplicationFromArgs(String clusterName,
+ AbstractClusterBuildingActionArgs args) throws IOException {
+ File file = args.getAppDef();
+ Path filePath = new Path(file.getAbsolutePath());
+ log.info("Loading app definition from: " + filePath);
+ Application application =
+ jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath);
+ if(args.lifetime > 0) {
+ application.setLifetime(args.lifetime);
+ }
+ application.setName(clusterName);
+ return application;
+ }
+
+ public int actionBuild(Application application) throws YarnException,
+ IOException {
+ Path appDir = checkAppNotExistOnHdfs(application);
+ persistApp(appDir, application);
+ return EXIT_SUCCESS;
+ }
public ApplicationId actionCreate(Application application)
throws IOException, YarnException {
@@ -684,8 +686,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2));
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>();
+ Map<String, LocalResource> localResources = new HashMap<>();
// copy local slideram-log4j.properties to hdfs and add to localResources
boolean hasSliderAMLog4j =
@@ -724,10 +725,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLaunchContext.setLocalResources(localResources);
addCredentialsIfSecure(conf, amLaunchContext);
submissionContext.setAMContainerSpec(amLaunchContext);
- yarnClient.submitApplication(submissionContext);
+ submitApplication(submissionContext);
return submissionContext.getApplicationId();
}
+ @VisibleForTesting
+ public ApplicationId submitApplication(ApplicationSubmissionContext context)
+ throws IOException, YarnException {
+ return yarnClient.submitApplication(context);
+ }
+
private void printLocalResources(Map<String, LocalResource> map) {
log.info("Added LocalResource for localization: ");
StringBuilder builder = new StringBuilder();
@@ -800,7 +807,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private Map<String, String> addAMEnv(Configuration conf, Path tempPath)
throws IOException {
- Map<String, String> env = new HashMap<String, String>();
+ Map<String, String> env = new HashMap<>();
ClasspathConstructor classpath =
buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib",
sliderFileSystem, getUsingMiniMRCluster());
@@ -929,69 +936,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return 0;
}
-// protected static void checkForCredentials(Configuration conf,
-// ConfTree tree, String clusterName) throws IOException {
-// if (tree.credentials == null || tree.credentials.isEmpty()) {
-// log.info("No credentials requested");
-// return;
-// }
-//
-// Console console = System.console();
-// for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
-// String provider = cred.getKey()
-// .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
-// .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
-// List<String> aliases = cred.getValue();
-// if (aliases == null || aliases.isEmpty()) {
-// continue;
-// }
-// Configuration c = new Configuration(conf);
-// c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
-// CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
-// Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
-// for (String alias : aliases) {
-// if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
-// log.info("Credentials for " + alias + " found in " + provider);
-// } else {
-// if (console == null) {
-// throw new IOException("Unable to input password for " + alias +
-// " because System.console() is null; provider " + provider +
-// " must be populated manually");
-// }
-// char[] pass = readPassword(alias, console);
-// credentialProvider.createCredentialEntry(alias, pass);
-// credentialProvider.flush();
-// Arrays.fill(pass, ' ');
-// }
-// }
-// }
-// }
-
- private static char[] readPassword(String alias, Console console)
- throws IOException {
- char[] cred = null;
-
- boolean noMatch;
- do {
- console.printf("%s %s: \n", PASSWORD_PROMPT, alias);
- char[] newPassword1 = console.readPassword();
- console.printf("%s %s again: \n", PASSWORD_PROMPT, alias);
- char[] newPassword2 = console.readPassword();
- noMatch = !Arrays.equals(newPassword1, newPassword2);
- if (noMatch) {
- if (newPassword1 != null) Arrays.fill(newPassword1, ' ');
- log.info(String.format("Passwords don't match. Try again."));
- } else {
- cred = newPassword1;
- }
- if (newPassword2 != null) Arrays.fill(newPassword2, ' ');
- } while (noMatch);
- if (cred == null)
- throw new IOException("Could not read credentials for " + alias +
- " from stdin");
- return cred;
- }
-
@Override
public int actionKeytab(ActionKeytabArgs keytabInfo)
throws YarnException, IOException {
@@ -1078,43 +1022,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
@Override
- public int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
- throws YarnException, IOException {
- log.warn("The 'install-keytab' option has been deprecated. Please use 'keytab --install'.");
- return actionKeytab(new ActionKeytabArgs(installKeytabInfo));
- }
-
- @Override
- public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
- YarnException,
- IOException {
- log.warn("The " + ACTION_INSTALL_PACKAGE
- + " option has been deprecated. Please use '"
- + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
- if (StringUtils.isEmpty(installPkgInfo.name)) {
- throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_TYPE_NAME + "\n"
- + CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
- }
- Path srcFile = extractPackagePath(installPkgInfo.packageURI);
-
- // Do not provide new options to install-package command as it is in
- // deprecated mode. So version is kept null here. Use package --install.
- Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name,
- null);
- FileSystem sfs = sliderFileSystem.getFileSystem();
- sfs.mkdirs(pkgPath);
-
- Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing package {} at {} and overwrite is {}.",
- srcFile, fileInFs, installPkgInfo.replacePkg);
- require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg),
- "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE);
- sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
- return EXIT_SUCCESS;
- }
-
- @Override
public int actionResource(ActionResourceArgs resourceInfo)
throws YarnException, IOException {
if (resourceInfo.help) {
@@ -1287,236 +1194,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
-
- @Override
- public int actionPackage(ActionPackageArgs actionPackageInfo)
- throws YarnException, IOException {
- initializeOutputStream(actionPackageInfo.out);
- int exitCode = -1;
- if (actionPackageInfo.help) {
- exitCode = actionHelp(ACTION_PACKAGE);
- }
- if (actionPackageInfo.install) {
- exitCode = actionPackageInstall(actionPackageInfo);
- }
- if (actionPackageInfo.delete) {
- exitCode = actionPackageDelete(actionPackageInfo);
- }
- if (actionPackageInfo.list) {
- exitCode = actionPackageList();
- }
- if (actionPackageInfo.instances) {
- exitCode = actionPackageInstances();
- }
- finalizeOutputStream(actionPackageInfo.out);
- if (exitCode != -1) {
- return exitCode;
- }
- throw new BadCommandArgumentsException(
- "Select valid package operation option");
- }
-
- private void initializeOutputStream(String outFile)
- throws IOException {
- if (outFile != null) {
- clientOutputStream = new PrintStream(outFile, "UTF-8");
- } else {
- clientOutputStream = System.out;
- }
- }
-
- private void finalizeOutputStream(String outFile) {
- if (outFile != null && clientOutputStream != null) {
- clientOutputStream.flush();
- clientOutputStream.close();
- }
- clientOutputStream = System.out;
- }
-
- private int actionPackageInstances() throws YarnException, IOException {
-// Map<String, Path> persistentInstances = sliderFileSystem
-// .listPersistentInstances();
-// if (persistentInstances.isEmpty()) {
-// log.info("No slider cluster specification available");
-// return EXIT_SUCCESS;
-// }
-// String pkgPathValue = sliderFileSystem
-// .buildPackageDirPath(StringUtils.EMPTY, StringUtils.EMPTY).toUri()
-// .getPath();
-// FileSystem fs = sliderFileSystem.getFileSystem();
-// Iterator<Map.Entry<String, Path>> instanceItr = persistentInstances
-// .entrySet().iterator();
-// log.info("List of applications with its package name and path");
-// println("%-25s %15s %30s %s", "Cluster Name", "Package Name",
-// "Package Version", "Application Location");
- //TODO deal with packages
-// while(instanceItr.hasNext()) {
-// Map.Entry<String, Path> entry = instanceItr.next();
-// String clusterName = entry.getKey();
-// Path clusterPath = entry.getValue();
-// AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
-// clusterName, clusterPath);
-// Path appDefPath = null;
-// try {
-// appDefPath = new Path(
-// getApplicationDefinitionPath(instanceDefinition
-// .getAppConfOperations()));
-// } catch (BadConfigException e) {
-// // Invalid cluster state, so move on to next. No need to log anything
-// // as this is just listing of instances.
-// continue;
-// }
-// if (!appDefPath.isUriPathAbsolute()) {
-// appDefPath = new Path(fs.getHomeDirectory(), appDefPath);
-// }
-// String appDefPathStr = appDefPath.toUri().toString();
-// try {
-// if (appDefPathStr.contains(pkgPathValue) && fs.isFile(appDefPath)) {
-// String packageName = appDefPath.getParent().getName();
-// String packageVersion = StringUtils.EMPTY;
-// if (instanceDefinition.isVersioned()) {
-// packageVersion = packageName;
-// packageName = appDefPath.getParent().getParent().getName();
-// }
-// println("%-25s %15s %30s %s", clusterName, packageName,
-// packageVersion, appDefPathStr);
-// }
-// } catch (IOException e) {
-// log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr);
-// }
-// }
- return EXIT_SUCCESS;
- }
-
- private int actionPackageList() throws IOException {
- Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY,
- StringUtils.EMPTY);
- log.info("Package install path : {}", pkgPath);
- FileSystem sfs = sliderFileSystem.getFileSystem();
- if (!sfs.isDirectory(pkgPath)) {
- log.info("No package(s) installed");
- return EXIT_SUCCESS;
- }
- FileStatus[] fileStatus = sfs.listStatus(pkgPath);
- boolean hasPackage = false;
- StringBuilder sb = new StringBuilder();
- sb.append("List of installed packages:\n");
- for (FileStatus fstat : fileStatus) {
- if (fstat.isDirectory()) {
- sb.append("\t").append(fstat.getPath().getName());
- sb.append("\n");
- hasPackage = true;
- }
- }
- if (hasPackage) {
- println(sb.toString());
- } else {
- log.info("No package(s) installed");
- }
- return EXIT_SUCCESS;
- }
-
- private void createSummaryMetainfoFile(Path srcFile, Path destFile,
- boolean overwrite) throws IOException {
- FileSystem srcFs = srcFile.getFileSystem(getConfig());
- try (InputStream inputStreamJson = SliderUtils
- .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json");
- InputStream inputStreamXml = SliderUtils
- .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) {
- InputStream inputStream = null;
- Path summaryFileInFs = null;
- if (inputStreamJson != null) {
- inputStream = inputStreamJson;
- summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
- + ".metainfo.json");
- log.info("Found JSON metainfo file in package");
- } else if (inputStreamXml != null) {
- inputStream = inputStreamXml;
- summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
- + ".metainfo.xml");
- log.info("Found XML metainfo file in package");
- }
- if (inputStream != null) {
- try (FSDataOutputStream dataOutputStream = sliderFileSystem
- .getFileSystem().create(summaryFileInFs, overwrite)) {
- log.info("Creating summary metainfo file");
- IOUtils.copy(inputStream, dataOutputStream);
- }
- }
- }
- }
-
- private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
- throws YarnException, IOException {
- requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
-
- Path srcFile = extractPackagePath(actionPackageArgs.packageURI);
-
- Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
- actionPackageArgs.version);
- FileSystem fs = sliderFileSystem.getFileSystem();
- if (!fs.exists(pkgPath)) {
- fs.mkdirs(pkgPath);
- }
-
- Path fileInFs = new Path(pkgPath, srcFile.getName());
- require(actionPackageArgs.replacePkg || !fs.exists(fileInFs),
- E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri());
-
- log.info("Installing package {} to {} (overwrite set to {})", srcFile,
- fileInFs, actionPackageArgs.replacePkg);
- fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
- createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg);
-
- String destPathWithHomeDir = Path
- .getPathWithoutSchemeAndAuthority(fileInFs).toString();
- String destHomeDir = Path.getPathWithoutSchemeAndAuthority(
- fs.getHomeDirectory()).toString();
- // a somewhat contrived approach to stripping out the home directory and any trailing
- // separator; designed to work on windows and unix
- String destPathWithoutHomeDir;
- if (destPathWithHomeDir.startsWith(destHomeDir)) {
- destPathWithoutHomeDir = destPathWithHomeDir.substring(destHomeDir.length());
- if (destPathWithoutHomeDir.startsWith("/") || destPathWithoutHomeDir.startsWith("\\")) {
- destPathWithoutHomeDir = destPathWithoutHomeDir.substring(1);
- }
- } else {
- destPathWithoutHomeDir = destPathWithHomeDir;
- }
- log.info("Set " + AgentKeys.APP_DEF + " in your app config JSON to {}",
- destPathWithoutHomeDir);
-
- return EXIT_SUCCESS;
- }
-
- private Path extractPackagePath(String packageURI)
- throws BadCommandArgumentsException {
- require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
- File pkgFile = new File(packageURI);
- require(pkgFile.isFile(),
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath());
- return new Path(pkgFile.toURI());
- }
-
- private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws
- YarnException, IOException {
- requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
-
- Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
- actionPackageArgs.version);
- FileSystem fs = sliderFileSystem.getFileSystem();
- require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri());
- log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath);
-
- if(fs.delete(pkgPath, true)) {
- log.info("Deleted package {} " + actionPackageArgs.name);
- return EXIT_SUCCESS;
- } else {
- log.warn("Package deletion failed.");
- return EXIT_NOT_FOUND;
- }
- }
-
@Override
public int actionUpdate(String clustername,
AbstractClusterBuildingActionArgs buildInfo) throws
@@ -1594,7 +1271,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String PLACEHOLDER_PATTERN = "\\$\\{[^{]+\\}";
Pattern placeholderPattern = Pattern.compile(PLACEHOLDER_PATTERN);
Matcher placeholderMatcher = placeholderPattern.matcher(env);
- Map<String, String> placeholderKeyValueMap = new HashMap<String, String>();
+ Map<String, String> placeholderKeyValueMap = new HashMap<>();
if (placeholderMatcher.find()) {
String placeholderKey = placeholderMatcher.group();
String systemKey = placeholderKey
@@ -1865,16 +1542,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
@VisibleForTesting
- public void actionFlex(String appName, ActionFlexArgs args)
+ public int actionFlex(String appName, ActionFlexArgs args)
throws YarnException, IOException {
- Component component = new Component();
- component.setNumberOfContainers(args.getNumberOfContainers());
- if (StringUtils.isEmpty(args.getComponent())) {
- component.setName("DEFAULT");
- } else {
- component.setName(args.getComponent());
+ Map<String, Long> componentCounts = new HashMap<>(args.getComponentMap()
+ .size());
+ for (Entry<String, String> entry : args.getComponentMap().entrySet()) {
+ long numberOfContainers = Long.parseLong(entry.getValue());
+ componentCounts.put(entry.getKey(), numberOfContainers);
}
- flex(appName, component);
+ // throw usage exception if no changes proposed
+ if (componentCounts.size() == 0) {
+ actionHelp(ACTION_FLEX);
+ }
+ flex(appName, componentCounts);
+ return EXIT_SUCCESS;
}
@Override
@@ -1966,19 +1647,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- @Override
- public String actionEcho(String name, ActionEchoArgs args) throws
- YarnException,
- IOException {
- String message = args.message;
- if (message == null) {
- throw new BadCommandArgumentsException("missing message");
- }
- SliderClusterOperations clusterOps =
- new SliderClusterOperations(bondToCluster(name));
- return clusterOps.echo(message);
- }
-
/**
* Find an instance of an application belonging to the current user.
* @param appname application name
@@ -2099,7 +1767,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
@Override
- public void actionStop(String appName, ActionFreezeArgs freezeArgs)
+ public int actionStop(String appName, ActionFreezeArgs freezeArgs)
throws YarnException, IOException {
validateClusterName(appName);
ApplicationReport app = findInstance(appName);
@@ -2112,7 +1780,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
.ordinal()) {
log.info("Application {} is in a terminated state {}", appName,
app.getYarnApplicationState());
- return;
+ return EXIT_SUCCESS;
}
try {
@@ -2127,6 +1795,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ " gracefully, forcefully kill the app.");
yarnClient.killApplication(app.getApplicationId(), freezeArgs.message);
}
+ return EXIT_SUCCESS;
}
@Override
@@ -2143,30 +1812,30 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return 0;
}
- public long flex(String appName, Component component)
- throws YarnException, IOException {
+ public Map<String, Long> flex(String appName, Map<String, Long>
+ componentCounts) throws YarnException, IOException {
validateClusterName(appName);
Path appDir = sliderFileSystem.buildClusterDirPath(appName);
Path appJson = new Path(appDir, appName + ".json");
Application persistedApp =
jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson);
- long original = 0;
- boolean foundComponent = false;
+ Map<String, Long> original = new HashMap<>(componentCounts.size());
for (Component persistedComp : persistedApp.getComponents()) {
- if (persistedComp.getName().equals(component.getName())) {
- original = persistedComp.getNumberOfContainers();
- persistedComp.setNumberOfContainers(component.getNumberOfContainers());
- foundComponent = true;
- break;
+ String name = persistedComp.getName();
+ if (componentCounts.containsKey(persistedComp.getName())) {
+ original.put(name, persistedComp.getNumberOfContainers());
+ persistedComp.setNumberOfContainers(componentCounts.get(name));
}
}
- if (!foundComponent) {
- throw new YarnException("Component " + component.getName()
- + " does not exist in app definition.");
+ if (original.size() < componentCounts.size()) {
+ componentCounts.keySet().removeAll(original.keySet());
+ throw new YarnException("Components " + componentCounts.keySet()
+ + " do not exist in app definition.");
}
jsonSerDeser
.save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true);
- log.info("Updated app definition file for component " + component);
+ log.info("Updated app definition file for components " + componentCounts
+ .keySet());
ApplicationReport instance = findInstance(appName);
if (instance != null) {
@@ -2174,11 +1843,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderClusterProtocol appMaster = connect(instance);
SliderClusterOperations clusterOps =
new SliderClusterOperations(appMaster);
- clusterOps.flex(component);
- log.info(
- "Application name = " + appName + ", Component name = " + component
- .getName() + ", number of containers updated from " + original
- + " to " + component.getNumberOfContainers());
+ clusterOps.flex(componentCounts);
+ for (Entry<String, Long> componentCount : componentCounts.entrySet()) {
+ log.info(
+ "Application name = " + appName + ", Component name = " +
+ componentCount.getKey() + ", number of containers updated " +
+ "from " + original.get(componentCount.getKey()) + " to " +
+ componentCount.getValue());
+ }
} else {
String message = "Application " + appName + "does not exist in RM. ";
throw new YarnException(message);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 2bb224b..197a564 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -20,26 +20,18 @@ package org.apache.slider.client;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDestroyArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionInstallKeytabArgs;
-import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
@@ -53,7 +45,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
-import java.util.Map;
/**
* Interface of those method calls in the slider API that are intended
@@ -63,8 +54,7 @@ import java.util.Map;
*/
public interface SliderClientAPI extends Service {
- void actionDestroy(String clustername) throws YarnException,
- IOException;
+ int actionDestroy(String clustername) throws YarnException, IOException;
/**
* AM to commit an asynchronous suicide
@@ -82,18 +72,6 @@ public interface SliderClientAPI extends Service {
throws SliderException;
/**
- * Upload keytab to a designated sub-directory of the user home directory
- *
- * @param installKeytabInfo the arguments needed to upload the keytab
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- * @deprecated use #actionKeytab
- */
- int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
- throws YarnException, IOException;
-
- /**
* Manage keytabs leveraged by slider
*
* @param keytabInfo the arguments needed to manage the keytab
@@ -105,17 +83,6 @@ public interface SliderClientAPI extends Service {
throws YarnException, IOException;
/**
- * Upload application package to user home directory
- *
- * @param installPkgInfo the arguments needed to upload the package
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
- int actionInstallPkg(ActionInstallPackageArgs installPkgInfo)
- throws YarnException, IOException;
-
- /**
* Manage file resources leveraged by slider
*
* @param resourceInfo the arguments needed to manage the resource
@@ -138,17 +105,6 @@ public interface SliderClientAPI extends Service {
throws IOException, YarnException;
/**
- * Managing slider application package
- *
- * @param pkgInfo the arguments needed to upload, delete or list the package
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
- int actionPackage(ActionPackageArgs pkgInfo)
- throws YarnException, IOException;
-
- /**
* Update the cluster specification
*
* @param clustername cluster name
@@ -179,7 +135,8 @@ public interface SliderClientAPI extends Service {
int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
- void actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
+ int actionFlex(String name, ActionFlexArgs args) throws YarnException,
+ IOException;
/**
* Test for a cluster existing probe for a cluster of the given name existing
@@ -200,17 +157,6 @@ public interface SliderClientAPI extends Service {
throws YarnException, IOException;
/**
- * Echo operation (not currently wired up to command line)
- * @param name cluster name
- * @param args arguments
- * @return the echoed text
- * @throws YarnException
- * @throws IOException
- */
- String actionEcho(String name, ActionEchoArgs args)
- throws YarnException, IOException;
-
- /**
* Status operation
*
* @param clustername cluster name
@@ -246,7 +192,7 @@ public interface SliderClientAPI extends Service {
* @param freezeArgs arguments to the stop
* @return EXIT_SUCCESS if the cluster was not running by the end of the operation
*/
- void actionStop(String clustername, ActionFreezeArgs freezeArgs)
+ int actionStop(String clustername, ActionFreezeArgs freezeArgs)
throws YarnException, IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index 3bb2af6..e89a660 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -25,7 +25,6 @@ import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
@@ -44,6 +43,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import static org.apache.slider.api.types.RestTypeMarshalling.unmarshall;
@@ -283,12 +284,17 @@ public class SliderClusterOperations {
return state;
}
- public void flex(Component component) throws IOException{
- Messages.FlexComponentRequestProto request =
- Messages.FlexComponentRequestProto.newBuilder()
- .setNumberOfContainers(component.getNumberOfContainers().intValue())
- .setName(component.getName()).build();
- appMaster.flexComponent(request);
+ public void flex(Map<String, Long> componentCounts) throws IOException{
+ Messages.FlexComponentsRequestProto.Builder builder =
+ Messages.FlexComponentsRequestProto.newBuilder();
+ for (Entry<String, Long> componentCount : componentCounts.entrySet()) {
+ Messages.ComponentCountProto componentProto =
+ Messages.ComponentCountProto.newBuilder()
+ .setName(componentCount.getKey())
+ .setNumberOfContainers(componentCount.getValue()).build();
+ builder.addComponents(componentProto);
+ }
+ appMaster.flexComponents(builder.build());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index b666834..e881edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -18,8 +18,6 @@
package org.apache.slider.common;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-
/**
* These are the keys that can be added to <code>conf/slider-client.xml</code>.
*/
@@ -105,23 +103,6 @@ public interface SliderXmlConfKeys {
*/
String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
- /**
- *
- * Use {@link RegistryConstants#KEY_REGISTRY_ZK_ROOT}
- *
- */
- @Deprecated
- String REGISTRY_PATH = "slider.registry.path";
-
- /**
- *
- * @Deprecated use {@link RegistryConstants#KEY_REGISTRY_ZK_QUORUM}
- *
- */
- @Deprecated
- String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum";
-
-
String IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH =
"ipc.client.fallback-to-simple-auth-allowed";
String HADOOP_HTTP_FILTER_INITIALIZERS =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index a2d4e38..c983a63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -19,6 +19,13 @@
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
/**
* Abstract Action to build things; shares args across build and
@@ -26,6 +33,14 @@ import com.beust.jcommander.Parameter;
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
+ @Parameter(names = {ARG_APPDEF},
+ description = "Template application definition file in JSON format.")
+ public File appDef;
+
+ public File getAppDef() {
+ return appDef;
+ }
+
@Parameter(names = {
ARG_QUEUE }, description = "Queue to submit the application")
public String queue;
@@ -33,4 +48,42 @@ public abstract class AbstractClusterBuildingActionArgs
@Parameter(names = {
ARG_LIFETIME }, description = "Lifetime of the application from the time of request")
public long lifetime;
+
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ @ParametersDelegate
+ public OptionArgsDelegate optionsDelegate =
+ new OptionArgsDelegate();
+
+
+ public Map<String, String> getOptionsMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getOptionsMap();
+ }
+
+ /**
+ * Get the role heap mapping (may be empty, but never null).
+ * @return role heap mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, Map<String, String>> getCompOptionMap() throws
+ BadCommandArgumentsException {
+ return optionsDelegate.getCompOptionMap();
+ }
+
+ @VisibleForTesting
+ public List<String> getComponentTuples() {
+ return componentDelegate.getComponentTuples();
+ }
+
+ /**
+ * Get the role mapping (may be empty, but never null).
+ * @return role mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws
+ BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java
new file mode 100644
index 0000000..57e4b02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_BUILD},
+ commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
+
+public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_BUILD;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
index c8cac65..4cc1077 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
@@ -18,25 +18,13 @@
package org.apache.slider.common.params;
-import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-import java.io.File;
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
- @Parameter(names = {ARG_APPDEF},
- description = "Template application definition file in JSON format.")
- public File appDef;
-
- public File getAppDef() {
- return appDef;
- }
-
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
deleted file mode 100644
index d05f10b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-
-public class ActionEchoArgs extends AbstractActionArgs {
- @Override
- public String getActionName() {
- return SliderActions.ACTION_ECHO;
- }
-
- @Parameter(names = {ARG_MESSAGE},
- description = "message to echo")
- public String message;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
index c565484..21cb609 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java
@@ -18,31 +18,38 @@
package org.apache.slider.common.params;
-import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.List;
+import java.util.Map;
@Parameters(commandNames = {SliderActions.ACTION_FLEX},
commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
public class ActionFlexArgs extends AbstractActionArgs {
- @Parameter(names = {ARG_COMPONENT},
- description = "component name")
- String componentName;
-
- @Parameter(names = {ARG_COUNT},
- description = "number of containers>")
- long numberOfContainers;
-
@Override
public String getActionName() {
return SliderActions.ACTION_FLEX;
}
- public String getComponent() {
- return componentName;
+ @ParametersDelegate
+ public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+
+ /**
+ * Get the component mapping (may be empty, but never null)
+ * @return mapping
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getComponentMap() throws
+ BadCommandArgumentsException {
+ return componentDelegate.getComponentMap();
}
- public long getNumberOfContainers() {
- return numberOfContainers;
+
+ public List<String> getComponentTuples() {
+ return componentDelegate.getComponentTuples();
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java
deleted file mode 100644
index 4cfb889..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallKeytabArgs.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = {SliderActions.ACTION_INSTALL_KEYTAB},
- commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_KEYTAB)
-
-public class ActionInstallKeytabArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_KEYTAB;
- }
-
- @Parameter(names = {ARG_KEYTAB},
- description = "Path to keytab on local disk")
- public String keytabUri;
-
- @Parameter(names = {ARG_FOLDER},
- description = "The name of the folder in which to store the keytab")
- public String folder;
-
- @Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
- public boolean overwrite = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 3;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
deleted file mode 100644
index 646e795..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-
-@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE)
-
-public class ActionInstallPackageArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_INSTALL_PACKAGE;
- }
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package on local disk")
- public String packageURI;
-
- @Parameter(names = {ARG_NAME},
- description = "The type of the package")
- public String name;
-
- @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package")
- public boolean replacePkg = false;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
index 32b1d2b..9a708ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionKeytabArgs.java
@@ -26,13 +26,6 @@ import com.beust.jcommander.Parameters;
public class ActionKeytabArgs extends AbstractActionArgs {
- public ActionKeytabArgs(ActionInstallKeytabArgs installKeytabInfo) {
- this.install = true;
- this.overwrite = installKeytabInfo.overwrite;
- this.keytab = installKeytabInfo.keytabUri;
- this.folder = installKeytabInfo.folder;
- }
-
public ActionKeytabArgs() {
super();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
deleted file mode 100644
index 4833934..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-
-@Parameters(commandNames = {SliderActions.ACTION_PACKAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE)
-
-public class ActionPackageArgs extends AbstractActionArgs {
-
- @Override
- public String getActionName() {
- return SliderActions.ACTION_PACKAGE;
- }
-
- @Parameter(names = {ARG_INSTALL},
- description = "Install package in the sub-folder 'package' of the user's Slider base directory")
- public boolean install;
-
- @Parameter(names = {ARG_PKGDELETE},
- description = "Delete package operation")
- public boolean delete;
-
- @Parameter(names = {ARG_PKGLIST},
- description = "List of package(s) installed")
- public boolean list;
-
- @Parameter(names = {ARG_PKGINSTANCES},
- description = "Lists all application instances referring to package")
- public boolean instances;
-
- @Parameter(names = {ARG_PACKAGE},
- description = "Path to app package on local disk")
- public String packageURI;
-
- @Parameter(names = {ARG_NAME},
- description = "Package name")
- public String name;
-
- @Parameter(names = {ARG_VERSION}, description = "Package version")
- public String version;
-
- @Parameter(names = {ARG_REPLACE_PKG},
- description = "Overwrite existing package")
- public boolean replacePkg = false;
-
- @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for package data")
- public String out;
-
- /**
- * Get the min #of params expected
- * @return the min number of params in the {@link #parameters} field
- */
- public int getMinParams() {
- return 0;
- }
-
- @Override
- public int getMaxParams() {
- return 1;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org