You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/01/29 18:25:53 UTC
incubator-slider git commit: SLIDER-752 No easy way to get list of
applications via API from SliderClient
Repository: incubator-slider
Updated Branches:
refs/heads/develop 3105ba9f3 -> 7e8903e36
SLIDER-752 No easy way to get list of applications via API from SliderClient
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e8903e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e8903e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e8903e3
Branch: refs/heads/develop
Commit: 7e8903e364ba02ba04efb6b99e153a1eb5120558
Parents: 3105ba9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jan 29 17:25:38 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jan 29 17:25:38 2015 +0000
----------------------------------------------------------------------
.../api/types/SliderInstanceDescription.java | 54 ++++++
.../org/apache/slider/client/SliderClient.java | 75 ++++----
.../apache/slider/client/SliderClientAPI.java | 22 +++
.../slider/client/SliderYarnClientImpl.java | 86 ++++++---
.../apache/slider/common/tools/SliderUtils.java | 37 ++++
.../slider/core/registry/YarnAppListClient.java | 93 +++++++++-
.../servicemonitor/YarnApplicationProbe.java | 2 +-
.../slider/agent/actions/TestActionList.groovy | 173 ++++++++++++-------
8 files changed, 410 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java b/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
new file mode 100644
index 0000000..3b95f80
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/SliderInstanceDescription.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+
+/**
+ * Description of a slider instance
+ */
+public class SliderInstanceDescription {
+
+ public final String name;
+ public final Path path;
+ public final ApplicationReport applicationReport;
+
+ public SliderInstanceDescription(String name,
+ Path path,
+ ApplicationReport applicationReport) {
+ this.name = name;
+ this.path = path;
+ this.applicationReport = applicationReport;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("SliderInstanceDescription{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", path=").append(path);
+ sb.append(", applicationReport: ")
+ .append(applicationReport == null
+ ? "null"
+ : (" id " + applicationReport.getApplicationId()));
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ace5f09..4dfbe4d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -70,6 +70,7 @@ import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
@@ -213,7 +214,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Yarn client service
*/
private SliderYarnClientImpl yarnClient;
- private YarnAppListClient YarnAppListClient;
+ private YarnAppListClient yarnAppListClient;
private AggregateConf launchedInstanceDefinition;
/**
@@ -457,7 +458,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnClient.start();
}
addService(yarnClient);
- YarnAppListClient =
+ yarnAppListClient =
new YarnAppListClient(yarnClient, getUsername(), getConfig());
// create the filesystem
sliderFileSystem = new SliderFileSystem(getConfig());
@@ -1993,10 +1994,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param user user: "" means all users, null means "default"
* @return a possibly empty list of Slider AMs
*/
- @VisibleForTesting
+
public List<ApplicationReport> listSliderInstances(String user)
throws YarnException, IOException {
- return YarnAppListClient.listInstances(user);
+ return yarnAppListClient.listInstances(user);
}
/**
@@ -2023,7 +2024,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* was named but it was not found
*/
@Override
- @VisibleForTesting
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
verifyBindingsDefined();
@@ -2092,7 +2092,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// list the details if all were requested, or the filtering contained
// a report
listed++;
- String details = instanceDetailsToString(name, report, verbose);
+ String details = SliderUtils.instanceDetailsToString(name,
+ report,
+ verbose);
print(details);
}
}
@@ -2101,40 +2103,27 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * Convert the instance details of an application to a string
- * @param name instance name
- * @param report the application report
- * @param verbose verbose output
- * @return a string
+ * Enumerate slider instances for the current user, and the
+ * most recent app report, where available.
+ * @param listOnlyInState boolean to indicate that the instances should
+ * only include those in a YARN state
+ * <code> minAppState <= currentState <= maxAppState </code>
+ *
+ * @param minAppState minimum application state to include in enumeration.
+ * @param maxAppState maximum application state to include
+ * @return a map of application instance name to description
+ * @throws IOException Any IO problem
+ * @throws YarnException YARN problems
*/
- String instanceDetailsToString(String name,
- ApplicationReport report,
- boolean verbose) {
- // format strings
- String staticf = "%-30s";
- String reportedf = staticf + " %10s %-40s";
- String livef = reportedf + " %s";
- StringBuilder builder = new StringBuilder(200);
- if (report == null) {
- builder.append(String.format(staticf, name));
- } else {
- // there's a report to look at
- String appId = report.getApplicationId().toString();
- String state = report.getYarnApplicationState().toString();
- if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
- // running: there's a URL
- builder.append(String.format(livef, name, state, appId ,report.getTrackingUrl()));
- } else {
- builder.append(String.format(reportedf, name, state, appId));
- }
- if (verbose) {
- builder.append('\n');
- builder.append(SliderUtils.appReportToString(report, "\n "));
- }
- }
-
- builder.append('\n');
- return builder.toString();
+ @Override
+ public Map<String, SliderInstanceDescription> enumSliderInstances(
+ boolean listOnlyInState,
+ YarnApplicationState minAppState,
+ YarnApplicationState maxAppState)
+ throws IOException, YarnException {
+ return yarnAppListClient.enumSliderInstances(listOnlyInState,
+ minAppState,
+ maxAppState);
}
/**
@@ -2233,7 +2222,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
} else {
// scan for instance in single --state state
- List<ApplicationReport> userInstances = yarnClient.listInstances("");
+ List<ApplicationReport> userInstances = yarnClient.listDeployedInstances("");
state = state.toUpperCase(Locale.ENGLISH);
YarnApplicationState desiredState = extractYarnApplicationState(state);
ApplicationReport foundInstance =
@@ -2300,7 +2289,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @return registry client -valid after the service is inited.
*/
public YarnAppListClient getYarnAppListClient() {
- return YarnAppListClient;
+ return yarnAppListClient;
}
/**
@@ -2312,7 +2301,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
private ApplicationReport findInstance(String appname)
throws YarnException, IOException {
- return YarnAppListClient.findInstance(appname);
+ return yarnAppListClient.findInstance(appname);
}
private RunningApplication findApplication(String appname)
@@ -2331,7 +2320,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private List<ApplicationReport> findAllLiveInstances(String appname)
throws YarnException, IOException {
- return YarnAppListClient.findAllLiveInstances(appname);
+ return yarnAppListClient.findAllLiveInstances(appname);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 328ec46..836891d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -23,7 +23,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
@@ -46,6 +48,7 @@ import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
import java.util.List;
+import java.util.Map;
/**
* Interface of those method calls in the slider API that are intended
@@ -170,6 +173,25 @@ public interface SliderClientAPI extends Service {
int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
/**
+ * Enumerate slider instances for the current user, and the
+ * most recent app report, where available.
+ * @param listOnlyInState boolean to indicate that the instances should
+ * only include those in a YARN state
+ * <code> minAppState <= currentState <= maxAppState </code>
+ *
+ * @param minAppState minimum application state to include in enumeration.
+ * @param maxAppState maximum application state to include
+ * @return a map of application instance name to description
+ * @throws IOException Any IO problem
+ * @throws YarnException YARN problems
+ */
+ Map<String, SliderInstanceDescription> enumSliderInstances(
+ boolean listOnlyInState,
+ YarnApplicationState minAppState,
+ YarnApplicationState maxAppState)
+ throws IOException, YarnException;
+
+ /**
* Implement the islive action: probe for a cluster of the given name existing
* @return exit code
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 856b34c..209169b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -18,8 +18,9 @@
package org.apache.slider.client;
-import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
@@ -32,7 +33,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.slf4j.Logger;
@@ -43,6 +46,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -55,6 +59,13 @@ public class SliderYarnClientImpl extends YarnClientImpl {
log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
/**
+ * Keyword to use in the {@link #emergencyForceKill(String)}
+ * operation to force kill <i>all</i> application instances belonging
+ * to a specific user
+ */
+ public static final String KILL_ALL = "all";
+
+ /**
* Get the RM Client RPC interface
* @return an RPC interface valid after initialization and authentication
*/
@@ -64,12 +75,28 @@ public class SliderYarnClientImpl extends YarnClientImpl {
/**
- * List Slider instances belonging to a specific user
+ * List Slider <i>running</i>instances belonging to a specific user.
+ * @deprecated use {@link #listDeployedInstances(String)}
* @param user user: "" means all users
* @return a possibly empty list of Slider AMs
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
+ return listDeployedInstances(user);
+ }
+
+ /**
+ * List Slider <i>deployed</i>instances belonging to a specific user.
+ * <p>
+ * Deployed means: known about in the YARN cluster; it will include
+ * any that are in the failed/finished state, as well as those queued
+ * for starting.
+ * @param user user: "" means all users
+ * @return a possibly empty list of Slider AMs
+ */
+ public List<ApplicationReport> listDeployedInstances(String user)
+ throws YarnException, IOException {
+ Preconditions.checkArgument(user != null, "Null User");
Set<String> types = new HashSet<String>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
@@ -84,18 +111,19 @@ public class SliderYarnClientImpl extends YarnClientImpl {
/**
- * find all instances of a specific app -if there is >1 in the cluster,
+ * find all instances of a specific app -if there is more than one in the
+ * YARN cluster,
* this returns them all
- * @param user user
+ * @param user user; use "" for all users
* @param appname application name
* @return the list of all matching application instances
*/
- @VisibleForTesting
public List<ApplicationReport> findAllInstances(String user,
- String appname) throws
- IOException,
- YarnException {
- List<ApplicationReport> instances = listInstances(user);
+ String appname)
+ throws IOException, YarnException {
+ Preconditions.checkArgument(appname != null, "Null application name");
+
+ List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport report : instances) {
@@ -113,6 +141,8 @@ public class SliderYarnClientImpl extends YarnClientImpl {
* @return true if the application is considered live
*/
public boolean isApplicationLive(ApplicationReport app) {
+ Preconditions.checkArgument(app != null, "Null app report");
+
return app.getYarnApplicationState().ordinal() <=
YarnApplicationState.RUNNING.ordinal();
}
@@ -120,15 +150,16 @@ public class SliderYarnClientImpl extends YarnClientImpl {
/**
* Kill a running application
- * @param applicationId
+ * @param applicationId app Id
+ * @param reason reason: reason for log
* @return the response
* @throws YarnException YARN problems
* @throws IOException IO problems
*/
public KillApplicationResponse killRunningApplication(ApplicationId applicationId,
- String reason) throws
- YarnException,
- IOException {
+ String reason)
+ throws YarnException, IOException {
+ Preconditions.checkArgument(applicationId != null, "Null application Id");
log.info("Killing application {} - {}", applicationId.getClusterTimestamp(),
reason);
KillApplicationRequest request =
@@ -140,19 +171,23 @@ public class SliderYarnClientImpl extends YarnClientImpl {
private String getUsername() throws IOException {
return UserGroupInformation.getCurrentUser().getShortUserName();
}
+
/**
* Force kill a yarn application by ID. No niceities here
+ * @param applicationId app Id. "all" means "kill all instances of the current user
+ *
*/
- public void emergencyForceKill(String applicationId) throws
- YarnException,
- IOException {
-
+ public void emergencyForceKill(String applicationId)
+ throws YarnException, IOException {
+
+ Preconditions.checkArgument(StringUtils.isNotEmpty(applicationId),
+ "Null/empty application Id");
- if ("all".equals(applicationId)) {
+ if (KILL_ALL.equals(applicationId)) {
// user wants all instances killed
String user = getUsername();
log.info("Killing all applications belonging to {}", user);
- Collection<ApplicationReport> instances = listInstances(user);
+ Collection<ApplicationReport> instances = listDeployedInstances(user);
for (ApplicationReport instance : instances) {
if (isApplicationLive(instance)) {
ApplicationId appId = instance.getApplicationId();
@@ -241,7 +276,9 @@ public class SliderYarnClientImpl extends YarnClientImpl {
String appname) throws
YarnException,
IOException {
- List<ApplicationReport> instances = listInstances(user);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
+ "Null/empty application name");
+ List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport app : instances) {
@@ -262,6 +299,9 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
+ Preconditions.checkArgument(instances != null, "Null instances list");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
+ "Null/empty application name");
// sort by most recent
SliderUtils.sortApplicationsByMostRecent(instances);
ApplicationReport found = null;
@@ -287,6 +327,10 @@ public class SliderYarnClientImpl extends YarnClientImpl {
public ApplicationReport findAppInInstanceList(List<ApplicationReport> instances,
String appname,
YarnApplicationState desiredState) {
+ Preconditions.checkArgument(instances != null, "Null instances list");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
+ "Null/empty application name");
+ Preconditions.checkArgument(desiredState != null, "Null desiredState");
ApplicationReport found = null;
ApplicationReport foundAndLive = null;
log.debug("Searching {} records for instance name {} in state '{}'",
@@ -307,6 +351,4 @@ public class SliderYarnClientImpl extends YarnClientImpl {
log.debug("No match");
return null;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index aeffe6c..3182bb7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -633,6 +633,43 @@ public final class SliderUtils {
return builder.toString();
}
+ /**
+ * Convert the instance details of an application to a string
+ * @param name instance name
+ * @param report the application report
+ * @param verbose verbose output
+ * @return a string
+ */
+ public static String instanceDetailsToString(String name,
+ ApplicationReport report,
+ boolean verbose) {
+ // format strings
+ String staticf = "%-30s";
+ String reportedf = staticf + " %10s %-40s";
+ String livef = reportedf + " %s";
+ StringBuilder builder = new StringBuilder(200);
+ if (report == null) {
+ builder.append(String.format(staticf, name));
+ } else {
+ // there's a report to look at
+ String appId = report.getApplicationId().toString();
+ String state = report.getYarnApplicationState().toString();
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
+ // running: there's a URL
+ builder.append(
+ String.format(livef, name, state, appId, report.getTrackingUrl()));
+ } else {
+ builder.append(String.format(reportedf, name, state, appId));
+ }
+ if (verbose) {
+ builder.append('\n');
+ builder.append(SliderUtils.appReportToString(report, "\n "));
+ }
+ }
+
+ builder.append('\n');
+ return builder.toString();
+ }
/**
* Sorts the given list of application reports, most recently started
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 6f50fca..1bdfb9c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -20,12 +20,21 @@ package org.apache.slider.core.registry;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Client code for interacting with a list of service instances.
@@ -33,9 +42,11 @@ import java.util.List;
*/
public class YarnAppListClient {
- final SliderYarnClientImpl yarnClient;
- final String username;
- final Configuration conf;
+ private final SliderYarnClientImpl yarnClient;
+ private final String username;
+ private final Configuration conf;
+ private static final Logger log =
+ LoggerFactory.getLogger(YarnAppListClient.class);
public YarnAppListClient(SliderYarnClientImpl yarnClient,
String username,
@@ -97,8 +108,82 @@ public class YarnAppListClient {
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
String listUser = user == null ? username : user;
- return yarnClient.listInstances(listUser);
+ return yarnClient.listDeployedInstances(listUser);
}
+ /**
+ * Enumerate slider instances for the current user, and the
+ * most recent app report, where available.
+ * @param listOnlyInState boolean to indicate that the instances should
+ * only include those in a YARN state
+ * <code> minAppState <= currentState <= maxAppState </code>
+ *
+ * @param minAppState minimum application state to include in enumeration.
+ * @param maxAppState maximum application state to include
+ * @return a map of application instance name to description
+ * @throws IOException Any IO problem
+ * @throws YarnException YARN problems
+ */
+ public Map<String, SliderInstanceDescription> enumSliderInstances(
+ boolean listOnlyInState,
+ YarnApplicationState minAppState,
+ YarnApplicationState maxAppState)
+ throws IOException, YarnException {
+
+ CoreFileSystem sliderFileSystem = new CoreFileSystem(conf);
+ Preconditions.checkArgument(!listOnlyInState || minAppState != null,
+ "null minAppState when listOnlyInState set");
+ Preconditions.checkArgument(!listOnlyInState || maxAppState != null,
+ "null maxAppState when listOnlyInState set");
+ if (!listOnlyInState) {
+ // if there's not filtering, ask for the entire range of states
+ minAppState = YarnApplicationState.NEW;
+ maxAppState = YarnApplicationState.KILLED;
+ }
+ // get the complete list of persistent instances
+ Map<String, Path> persistentInstances =
+ sliderFileSystem.listPersistentInstances();
+ Map<String, SliderInstanceDescription> descriptions =
+ new HashMap<String, SliderInstanceDescription>(persistentInstances.size());
+
+ if (persistentInstances.isEmpty()) {
+ // an empty listing is a success if no cluster was named
+ log.debug("No application instances found");
+ return descriptions;
+ }
+
+ // enum those the RM knows about
+ List<ApplicationReport> rmInstances = listInstances();
+ SliderUtils.sortApplicationsByMostRecent(rmInstances);
+ Map<String, ApplicationReport> reportMap =
+ SliderUtils.buildApplicationReportMap(rmInstances, minAppState,
+ maxAppState);
+ log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
+ persistentInstances.size(),
+ rmInstances.size(),
+ minAppState, maxAppState,
+ reportMap.size());
+
+ // at this point there is a list of all persistent instances, and
+ // a (possibly filtered) list of application reports
+
+ for (Map.Entry<String, Path> entry : persistentInstances.entrySet()) {
+ // loop through the persistent values
+ String name = entry.getKey();
+
+ // look up any report from the (possibly filtered) report set
+ ApplicationReport report = reportMap.get(name);
+ if (!listOnlyInState || report != null) {
+ // if the enum wants to filter in state, only add it if there is
+ // a report in that range. Otherwise: include all values
+ SliderInstanceDescription sid = new SliderInstanceDescription(
+ name, entry.getValue(), report);
+ descriptions.put(name, sid);
+ }
+ }
+
+ return descriptions;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
index 8bc6dd0..adf613c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
@@ -73,7 +73,7 @@ public class YarnApplicationProbe extends Probe {
try {
List<ApplicationReport> instances =
- yarnClient.listInstances(username);
+ yarnClient.listDeployedInstances(username);
ApplicationReport instance =
yarnClient.findClusterInInstanceList(instances, clustername);
if (null == instance) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e8903e3/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index bf65b0f..15ad701 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.agent.actions
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -55,35 +56,17 @@ class TestActionList extends AgentMiniClusterTestBase {
@Test
public void testActionListSuite() throws Throwable {
testListThisUserNoClusters()
- testListLiveCluster()
testListMissingCluster()
- testActionListStates()
- }
-
- public void testListThisUserNoClusters() throws Throwable {
- log.info("RM address = ${RMAddr}")
- ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
- //config includes RM binding info
- new YarnConfiguration(miniCluster.config),
- //varargs list of command line params
- [
- SliderActions.ACTION_LIST,
- Arguments.ARG_MANAGER, RMAddr
- ]
- )
- assert launcher.serviceExitCode == 0
+ testActionList()
}
- public void testListLiveCluster() throws Throwable {
- //launch the cluster
- String clustername = "testlistlivecluster"
+ public void testActionList() {
+ String clustername = "testactionlist"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
- false)
-
+ true)
addToTeardown(launcher)
- //do the low level operations to get a better view of what is going on
SliderClient sliderClient = launcher.service
waitForClusterLive(sliderClient)
@@ -98,12 +81,13 @@ class TestActionList extends AgentMiniClusterTestBase {
)
assert launcher.serviceExitCode == 0
//now look for the explicit sevice
-
+
def serviceRegistryClient = sliderClient.yarnAppListClient
ApplicationReport instance = serviceRegistryClient.findInstance(clustername)
assert instance != null
log.info(instance.toString())
+ ApplicationId originalAppId = instance.applicationId;
//now list with the named cluster
launcher = launchClientAgainstMiniMR(
@@ -114,52 +98,36 @@ class TestActionList extends AgentMiniClusterTestBase {
SliderActions.ACTION_LIST, clustername
]
)
- clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
- }
- public void testListMissingCluster() throws Throwable {
- describe("exec the list command against an unknown cluster")
+ describe "listing by state"
+ //Listing only live instances
+ assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0;
+ assert sliderClient.actionList(clustername,
+ new ActionListArgs(live: true, verbose:true)) == 0;
- ServiceLauncher<SliderClient> launcher
- try {
- launcher = launchClientAgainstMiniMR(
- //config includes RM binding info
- new YarnConfiguration(miniCluster.config),
- //varargs list of command line params
- [
- SliderActions.ACTION_LIST,
- "no-instance"
- ]
- )
- fail("expected an exception, got a status code " + launcher.serviceExitCode)
- } catch (UnknownApplicationInstanceException e) {
- //expected
- }
- }
+ // find the same via the low-level operations
- public void testActionListStates() {
- String clustername = "testactionliststates"
- ServiceLauncher<SliderClient> launcher = createStandaloneAM(
- clustername,
- true,
- true)
- addToTeardown(launcher)
- SliderClient sliderClient = launcher.service
- waitForClusterLive(sliderClient)
+ def instances = sliderClient.enumSliderInstances(false, null, null)
+ assert instances.size() > 0
+ def enumeratedInstance = instances[clustername]
+ assert enumeratedInstance.name == clustername
+ assert enumeratedInstance.path.toString().endsWith("/" + clustername)
+ assert enumeratedInstance.applicationReport != null
+ assert originalAppId == enumeratedInstance.applicationReport.applicationId
+ assert enumeratedInstance.applicationReport.yarnApplicationState == YarnApplicationState.RUNNING
+
+ instances = sliderClient.enumSliderInstances(true,
+ YarnApplicationState.RUNNING, YarnApplicationState.RUNNING)
+ assert instances[clustername]
- describe "listing"
- //Listing only live instances
- assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0;
- assert sliderClient.actionList(clustername,
- new ActionListArgs(live: true, verbose:true)) == 0;
clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
waitForAppToFinish(sliderClient)
-
+
try {
// unknown yarn state
- int e= sliderClient.actionList(clustername,
+ int e = sliderClient.actionList(clustername,
new ActionListArgs(state: "undefined"));
fail("expected failure, got return code of $e")
} catch (BadCommandArgumentsException expected) {
@@ -168,12 +136,13 @@ class TestActionList extends AgentMiniClusterTestBase {
try {
// state and --live options
- int e= sliderClient.actionList(clustername,
+ int e = sliderClient.actionList(clustername,
new ActionListArgs(state: "running", live: true));
fail("expected failure, got return code of $e")
} catch (BadCommandArgumentsException expected) {
-
+ // expected
}
+
//Listing only live instances but prints nothing since instance is frozen/stopped
describe("after freeze")
@@ -194,6 +163,21 @@ class TestActionList extends AgentMiniClusterTestBase {
assert -1 == sliderClient.actionList("",
new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+ // now look for finished app state
+ instances = sliderClient.enumSliderInstances(false, null, null)
+ enumeratedInstance = instances[clustername]
+ assert enumeratedInstance.applicationReport.yarnApplicationState ==
+ YarnApplicationState.FINISHED
+ // look for running apps, expect no match
+ instances = sliderClient.enumSliderInstances(true,
+ YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING)
+ assert null == instances[clustername]
+
+ // look for terminated apps, expect no match
+ instances = sliderClient.enumSliderInstances(true,
+ YarnApplicationState.FINISHED, YarnApplicationState.KILLED)
+ assert instances[clustername]
+
// thaw
sliderClient.actionThaw(clustername, new ActionThawArgs());
waitForClusterLive(sliderClient)
@@ -201,7 +185,7 @@ class TestActionList extends AgentMiniClusterTestBase {
describe("Post-thaw listing")
assert 0 == sliderClient.actionList(clustername,
new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
-
+
//Listing only live instances
assert 0 == sliderClient.actionList(clustername,
new ActionListArgs(live: true));
@@ -209,9 +193,74 @@ class TestActionList extends AgentMiniClusterTestBase {
//Listing all the instance both history (previously freezed instance) and live
assert 0 == sliderClient.actionList("", new ActionListArgs(live: true));
+ // look for terminated apps, expect no match
+ instances = sliderClient.enumSliderInstances(true,
+ YarnApplicationState.RUNNING, YarnApplicationState.RUNNING)
+ assert instances[clustername]
+ def runningId = instances[clustername].applicationReport.applicationId
+ assert runningId != originalAppId
+
+ // stop again
+
maybeStopCluster(sliderClient, "", "forced", true)
assert 0 == sliderClient.actionList(clustername,
new ActionListArgs(state: "killed"));
+
+ // look for terminated apps, match
+ instances = sliderClient.enumSliderInstances(true,
+ YarnApplicationState.FINISHED, YarnApplicationState.KILLED)
+ assert instances[clustername]
+
+ // and verify the report picked up is the latest one
+ def finishedInstance = instances[clustername]
+
+ def finishedAppReport = finishedInstance.applicationReport
+ assert runningId == finishedAppReport.applicationId
+ // which was force killed
+ assert YarnApplicationState.KILLED == finishedAppReport.yarnApplicationState
+
+ // check that an enum for live apps fails
+ assert 0 == sliderClient.enumSliderInstances(true,
+ YarnApplicationState.RUNNING, YarnApplicationState.RUNNING).size()
+
+ // check that an enum for non-live apps works
+ assert 0 < sliderClient.enumSliderInstances(false,
+ YarnApplicationState.RUNNING, YarnApplicationState.RUNNING).size()
+ }
+
+
+
+ public void testListThisUserNoClusters() throws Throwable {
+ log.info("RM address = ${RMAddr}")
+ ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ Arguments.ARG_MANAGER, RMAddr
+ ]
+ )
+ assert launcher.serviceExitCode == 0
+ }
+ public void testListMissingCluster() throws Throwable {
+ describe("exec the list command against an unknown cluster")
+
+ ServiceLauncher<SliderClient> launcher
+ try {
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_LIST,
+ "no-instance"
+ ]
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (UnknownApplicationInstanceException e) {
+ //expected
+ }
}
}