You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/24 22:51:29 UTC
[8/9] git commit: SLIDER-306 new list options to get full persistent
list, live clusters, clusters in given states
SLIDER-306 new list options to get full persistent list, live clusters, clusters in given states
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd8384c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd8384c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd8384c9
Branch: refs/heads/develop
Commit: bd8384c90ca7d70d4b6969bc18082df6dbbd0d9c
Parents: b19e157
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 24 21:46:23 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 24 21:51:10 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 200 ++++++++-----
.../apache/slider/client/SliderClientAPI.java | 5 +-
.../slider/common/params/ActionListArgs.java | 10 +-
.../common/params/ActionRegistryArgs.java | 1 +
.../apache/slider/common/params/Arguments.java | 1 -
.../slider/common/tools/CoreFileSystem.java | 30 ++
.../apache/slider/common/tools/SliderUtils.java | 290 +++++++++++--------
.../apache/slider/core/build/InstanceIO.java | 8 +-
.../slider/core/registry/YarnAppListClient.java | 21 +-
.../slider/agent/actions/TestActionList.groovy | 98 ++++---
.../standalone/TestStandaloneAMDestroy.groovy | 2 +-
.../funtest/framework/CommandTestBase.groovy | 10 +-
.../funtest/commands/ListCommandIT.groovy | 14 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 26 +-
.../funtest/HBaseClusterLifecycleIT.groovy | 2 +-
15 files changed, 456 insertions(+), 262 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 4ac4031..66797ae 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathNotFoundException;
@@ -1666,100 +1667,162 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * List Slider instances belonging to a specific user
- * @param user user: "" means all users
+ * List Slider instances belonging to a specific user. This will include
+ * failed and killed instances; there may be duplicates
+ * @param user user: "" means all users, null means "default"
* @return a possibly empty list of Slider AMs
*/
@VisibleForTesting
public List<ApplicationReport> listSliderInstances(String user)
throws YarnException, IOException {
- return YarnAppListClient.listInstances();
+ return YarnAppListClient.listInstances(user);
}
- @Override
/**
- * Implement the list action: list all nodes
- *
- * live: List out only live instances
- * history: List out only history instances
- *
- * If arguments are not given then list out both finished and
- * running instances
- *
- * @param clustername List out specific cluster
- * @param args Action list arguments
- * @return exit code of 0 if a list was created
+ * A basic list action to list live instances
+ * @param clustername cluster name
+ * @return success if the listing was considered successful
+ * @throws IOException
+ * @throws YarnException
*/
+ public int actionList(String clustername) throws IOException, YarnException {
+ ActionListArgs args = new ActionListArgs();
+ args.live = true;
+ return actionList(clustername, args);
+ }
+
+ /**
+ * Implement the list action: list all nodes
+
+ * @param clustername List out specific instance name
+ * @param args Action list arguments
+ * @return 0 if one or more entries were listed
+ */
+ @Override
@VisibleForTesting
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
verifyBindingsDefined();
- String user = UserGroupInformation.getCurrentUser().getUserName();
- List<ApplicationReport> instances = listSliderInstances(user);
- SliderUtils.sortApplicationReport(instances);
boolean live = args.live;
- boolean history = args.history;
- if (isUnset(clustername)) {
- // no cluster name: list all
- log.info("Instances for {}: {}",
- (user != null ? user : "all users"),
- instances.size());
- for (ApplicationReport report : instances) {
- logAppReport(report, live, history);
- }
- // and always succeed
+ String state = args.state;
+ boolean verbose = args.verbose;
+
+ if (live && !state.isEmpty()) {
+ throw new BadCommandArgumentsException(
+ Arguments.ARG_LIVE + " and " + Arguments.ARG_STATE + " are exclusive");
+ }
+ // flag to indicate only services in a specific state are to be listed
+ boolean listOnlyInState = live || !state.isEmpty();
+
+ YarnApplicationState min, max;
+ if (live) {
+ min = YarnApplicationState.NEW;
+ max = YarnApplicationState.RUNNING;
+ } else if (!state.isEmpty()) {
+ YarnApplicationState stateVal = extractYarnApplicationState(state);
+ min = max = stateVal;
+ } else {
+ min = YarnApplicationState.NEW;
+ max = YarnApplicationState.KILLED;
+ }
+ // get the complete list of persistent instances
+ Map<String, Path> persistentInstances = sliderFileSystem.listPersistentInstances();
+
+ if (persistentInstances.isEmpty() && isUnset(clustername)) {
+ // an empty listing is a success if no cluster was named
+ log.debug("No application instances found");
return EXIT_SUCCESS;
+ }
+
+ // and those the RM knows about
+ List<ApplicationReport> instances = listSliderInstances(null);
+ SliderUtils.sortApplicationReport(instances);
+ Map<String, ApplicationReport> reportMap =
+ SliderUtils.buildApplicationReportMap(instances, min, max);
+ log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
+ persistentInstances.size(),
+ instances.size(),
+ min, max,
+ reportMap.size() );
+
+ if (isSet(clustername)) {
+ // only one instance is expected
+ // resolve the persistent value
+ Path persistent = persistentInstances.get(clustername);
+ if (persistent == null) {
+ throw unknownClusterException(clustername);
+ }
+ persistentInstances = new HashMap<String, Path>();
+ persistentInstances.put(clustername, persistent);
+ }
+
+ // at this point there is either the entire list or a stripped down instance
+ int listed = 0;
+
+ for (String name : persistentInstances.keySet()) {
+ ApplicationReport report = reportMap.get(name);
+ if (!listOnlyInState || report != null) {
+ // list the details if all were requested, or the filtering contained
+ // a report
+ listed++;
+ String details = instanceDetailsToString(name, report, verbose);
+ print(details);
+ }
+ }
+
+ return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE;
+ }
+
+ String instanceDetailsToString(String name,
+ ApplicationReport report,
+ boolean verbose) {
+ // format strings
+ String staticf = "%-30s";
+ String reportedf = staticf + " %10s %-40s";
+ String livef = reportedf + " %s";
+ StringBuilder builder = new StringBuilder(200);
+ if (report == null) {
+ builder.append(String.format(staticf, name));
} else {
- // cluster name provided
- SliderUtils.validateClusterName(clustername);
- log.debug("Listing cluster named {}, live={}, history={}",
- clustername, live, history);
- boolean instanceFound = false;
- if (history) {
- for (ApplicationReport report : instances) {
- if (report.getName().equals(clustername)) {
- logAppReport(report, live, true);
- instanceFound = true;
- }
- }
+ // there's a report to look at
+ String appId = report.getApplicationId().toString();
+ String state = report.getYarnApplicationState().toString();
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
+ // running: there's a URL
+ builder.append(String.format(livef, name, state, appId ,report.getTrackingUrl()));
} else {
- // no history flag, only list live value
- ApplicationReport report =
- findClusterInInstanceList(instances, clustername);
- if (report != null) {
- instanceFound = logAppReport(report, true, false);
- }
+ builder.append(String.format(reportedf, name, state, appId));
}
- // exit code if the instance was found
- if (instanceFound) {
- return EXIT_SUCCESS;
- } else {
- throw unknownClusterException(clustername);
+ if (verbose) {
+ builder.append('\n');
+ builder.append(SliderUtils.appReportToString(report, "\n "));
}
}
+
+ builder.append('\n');
+ return builder.toString();
}
/**
- * Log the application report at INFO
- * @param report report to log
- * @param live only list live apps
- * @param history list historical containers
- * @return whether the report was logged or not
+ * Extract the state of a Yarn application --state argument
+ * @param state state argument
+ * @return the application state
+ * @throws BadCommandArgumentsException if the argument did not match
+ * any known state
*/
- public boolean logAppReport(ApplicationReport report,
- boolean live,
- boolean history) {
- // app is active if it is accepted or running
- boolean active = isApplicationActive(report);
-
- boolean toLog = (active && live) || (!active && history);
- if (toLog) {
- log.info(SliderUtils.appReportToString(report, "\n"));
+ private YarnApplicationState extractYarnApplicationState(String state) throws
+ BadCommandArgumentsException {
+ YarnApplicationState stateVal;
+ try {
+ stateVal = YarnApplicationState.valueOf(state.toUpperCase(Locale.ENGLISH));
+ } catch (IllegalArgumentException e) {
+ throw new BadCommandArgumentsException("Unknown state: " + state);
+
}
- return toLog;
+ return stateVal;
}
-
+
/**
* Is an application active: accepted or running
* @param report the application report
@@ -1839,8 +1902,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// scan for instance in single --state state
List<ApplicationReport> userInstances = yarnClient.listInstances("");
state = state.toUpperCase(Locale.ENGLISH);
- YarnApplicationState desiredState =
- YarnApplicationState.valueOf(state);
+ YarnApplicationState desiredState = extractYarnApplicationState(state);
ApplicationReport foundInstance =
yarnClient.findAppInInstanceList(userInstances, name, desiredState);
if (foundInstance != null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 09ca8fa..61948e3 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -19,6 +19,7 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -32,6 +33,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
@@ -41,6 +43,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
+import java.util.List;
/**
* Interface of those method calls in the slider API that are intended
@@ -138,7 +141,7 @@ public interface SliderClientAPI extends Service {
* Implement the list action: list all nodes
* @return exit code of 0 if a list was created
*/
- int actionList(String clustername) throws IOException, YarnException;
+ int actionList(String clustername, ActionListArgs args) throws IOException, YarnException;
/**
* Implement the islive action: probe for a cluster of the given name existing
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 586f0a1..30cc93e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -34,9 +34,13 @@ public class ActionListArgs extends AbstractActionArgs {
description = "List only live application instances")
public boolean live;
- @Parameter(names = {ARG_HISTORY},
- description = "List only historical application instances")
- public boolean history;
+ @Parameter(names = {ARG_STATE},
+ description = "list only applications in the specific YARN state")
+ public String state = "";
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
/**
* Get the min #of params expected
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index 36ba341..4b4c30a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -137,6 +137,7 @@ public class ActionRegistryArgs extends AbstractActionArgs {
@Parameter(names = {ARG_USER},
description = "the name of the user whose application is being resolved")
public String user;
+
/**
* validate health of all the different operations
* @throws BadCommandArgumentsException
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 693ad2f..06d9dfb 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -55,7 +55,6 @@ public interface Arguments {
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HELP = "--help";
- String ARG_HISTORY = "--history";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INTERNAL = "--internal";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 23eeefe..7377dd3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -555,6 +555,36 @@ public class CoreFileSystem {
return builder.toString();
}
+ /**
+ * List all application instances persisted for this user, giving the
+ * patha. The instance name is the last element in the path
+ * @return a possibly empty map of application instance names to paths
+ */
+ public Map<String, Path> listPersistentInstances() throws IOException {
+ FileSystem fs = getFileSystem();
+ Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
+ if (!fs.exists(path)) {
+ // special case: no instances have ever been created
+ return new HashMap<String, Path>(0);
+ }
+ FileStatus[] statuses = fs.listStatus(path);
+ Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
+
+ // enum the child entries
+ for (FileStatus status : statuses) {
+ if (status.isDirectory()) {
+ // for directories, look for an internal.json underneath
+ Path child = status.getPath();
+ Path internalJson = new Path(child, Filenames.INTERNAL);
+ if (fs.exists(internalJson)) {
+ // success => this is an instance
+ instances.put(child.getName(), child);
+ }
+ }
+ }
+ return instances;
+ }
+
public void touch(Path path, boolean overwrite) throws IOException {
FSDataOutputStream out = fileSystem.create(path, overwrite);
out.close();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 106a052..a51366e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -97,6 +97,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -513,7 +514,7 @@ public final class SliderUtils {
int length = separator.length();
String s = b.toString();
return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
+ s : (b.substring(0, b.length() - length));
}
/**
@@ -631,12 +632,15 @@ public final class SliderUtils {
if (instances.size() <= 1) {
return;
}
- List<ApplicationReport> nonLiveInstance = new ArrayList<ApplicationReport>(instances.size());
- List<ApplicationReport> liveInstance = new ArrayList<ApplicationReport>(instances.size());
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<ApplicationReport>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport report : instances) {
if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
- || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
liveInstance.add(report);
} else {
nonLiveInstance.add(report);
@@ -644,21 +648,23 @@ public final class SliderUtils {
}
if (liveInstance.size() > 1) {
- Comparator<ApplicationReport> liveInstanceComparator = new Comparator<ApplicationReport>() {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- return Long.compare(r1.getStartTime(), r2.getStartTime());
- }
- };
+ Comparator<ApplicationReport> liveInstanceComparator =
+ new Comparator<ApplicationReport>() {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ return Long.compare(r1.getStartTime(), r2.getStartTime());
+ }
+ };
Collections.sort(liveInstance, liveInstanceComparator);
}
if (nonLiveInstance.size() > 1) {
- Comparator<ApplicationReport> nonLiveInstanceComparator = new Comparator<ApplicationReport>() {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- return Long.compare(r1.getFinishTime(), r2.getFinishTime());
- }
- };
+ Comparator<ApplicationReport> nonLiveInstanceComparator =
+ new Comparator<ApplicationReport>() {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ return Long.compare(r1.getFinishTime(), r2.getFinishTime());
+ }
+ };
Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
}
instances.clear();
@@ -667,6 +673,32 @@ public final class SliderUtils {
}
/**
+ * Built a (sorted) map of application reports, mapped to the instance name
+ * The list is sorted, and the addition process does not add a report
+ * if there is already one that exists. If the list handed in is sorted,
+ * those that are listed first form the entries returned
+ * @param instances list of intances
+ * @param minState minimum YARN state to be included
+ * @param maxState maximum YARN state to be included
+ * @return all reports in the list whose state >= minimum and <= maximum
+ */
+ public static Map<String, ApplicationReport> buildApplicationReportMap(
+ List<ApplicationReport> instances,
+ YarnApplicationState minState, YarnApplicationState maxState) {
+ TreeMap<String, ApplicationReport> map = new TreeMap<String, ApplicationReport>();
+ for (ApplicationReport report : instances) {
+ YarnApplicationState state = report.getYarnApplicationState();
+ if (state.ordinal() >= minState.ordinal() &&
+ state.ordinal() <= maxState.ordinal() &&
+ map.get(report.getName()) == null) {
+ map.put(report.getName(), report);
+ }
+ }
+ return map;
+ }
+
+
+ /**
* Merge in one map to another -all entries in the second map are
* merged into the first -overwriting any duplicate keys.
* @param first first map -the updated one.
@@ -1079,7 +1111,8 @@ public final class SliderUtils {
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
" -security has already been set up with the wrong authentication method. "
- + "This can occur if a file system has already been created prior to the loading of "
+ +
+ "This can occur if a file system has already been created prior to the loading of "
+ "the security configuration.",
authUser);
@@ -1415,7 +1448,7 @@ public final class SliderUtils {
String appHomeOption =
internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
- if(!isUnset(appHomeOption)) {
+ if (!isUnset(appHomeOption)) {
throw new BadClusterStateException(
ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
}
@@ -1627,7 +1660,9 @@ public final class SliderUtils {
* @throws FileNotFoundException if the file is not considered valid
* @param logger
*/
- public static void maybeVerifyWinUtilsValid(Logger logger) throws IOException, SliderException {
+ public static void maybeVerifyWinUtilsValid(Logger logger) throws
+ IOException,
+ SliderException {
if (!Shell.WINDOWS) {
return;
}
@@ -1668,19 +1703,19 @@ public final class SliderUtils {
* Look for the windows executable and check it has the right headers.
* <code>File.canRead()</code> doesn't work on windows, so the reading
* is mandatory.
- *
+ *
* @param program program name for errors
* @param exe executable
* @throws IOException IOE
*/
- public static void verifyWindowsExe(String program, File exe)
+ public static void verifyWindowsExe(String program, File exe)
throws IOException {
verifyIsFile(program, exe);
verifyFileSize(program, exe, 0x100);
// now read two bytes and verify the header.
-
+
FileReader reader = null;
try {
int[] header = new int[2];
@@ -1811,7 +1846,7 @@ public final class SliderUtils {
}
}
}
- if (errorText== null) {
+ if (errorText == null) {
return process;
}
@@ -1862,107 +1897,110 @@ public final class SliderUtils {
execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "--version");
}
- /**
- * return the path to the currently running slider command
- *
- * @throws NullPointerException
- * - If the pathname argument is null
- * @throws SecurityException
- * - if a security manager exists and its checkPermission method
- * doesn't allow getting the ProtectionDomain
- */
- public static String getCurrentCommandPath() {
- File f = new File(Slider.class.getProtectionDomain().getCodeSource()
- .getLocation().getPath());
- return f.getAbsolutePath();
- }
-
- /**
- * return the path to the slider-client.xml used by the current running
- * slider command
- *
- * @throws SecurityException
- * - if a security manager exists and its checkPermission method
- * denies access to the class loader for the class
- */
- public static String getClientConfigPath() {
- URL path = ConfigHelper.class.getClassLoader().getResource(
- SliderKeys.CLIENT_RESOURCE);
- return path.toString();
- }
-
- /**
- * validate if slider-client.xml under the path can be opened
- *
- * @throws IOException
- * : the file can't be found or open
- */
- public static void validateClientConfigFile() throws IOException {
- URL resURL = SliderVersionInfo.class.getClassLoader().getResource(
- SliderKeys.CLIENT_RESOURCE);
- if (resURL == null) {
- throw new IOException(
- "slider-client.xml doesn't exist on the path: "
- + getClientConfigPath());
- }
-
- try {
- InputStream inStream = resURL.openStream();
- if (inStream == null) {
- throw new IOException("slider-client.xml can't be opened");
- }
- } catch (IOException e) {
- throw new IOException("slider-client.xml can't be opened: "
- + e.toString());
- }
- }
-
- /**
- * validate if a file on HDFS can be open
- *
- * @throws IOException the file can't be found or opened
- * @throws URISyntaxException
- */
- public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr)
- throws IOException, URISyntaxException{
- URI pathURI = new URI(pathStr);
- InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(pathURI));
- if(inputStream == null){
- throw new IOException("HDFS file " + pathStr + " can't be opened");
- }
- }
-
- /**
- * return the version and path of the JDK invoking the current running
- * slider command
- *
- * @throws SecurityException
- * - if a security manager exists and its checkPropertyAccess
- * method doesn't allow access to the specified system property.
- */
- public static String getJDKInfo() {
- String version = System.getProperty("java.version");
- String javaHome = System.getProperty("java.home");
- return "The version of the JDK invoking the current running slider command: "
- + version + "; The path to it is: " + javaHome;
- }
-
- /**
- * return a description of whether the current user has created credential
- * cache files from kerberos servers
- *
- * @throws IOException
- * @throws BadConfigException
- * @throws SecurityException
- * - if a security manager exists and its checkPropertyAccess
- * method doesn't allow access to the specified system property.
- */
- public static String checkCredentialCacheFile() throws IOException,
- BadConfigException {
- String result = null;
- if (!Shell.WINDOWS) {
- result = Shell.execCommand("klist");
- }
- return result;
- }
+ /**
+ * return the path to the currently running slider command
+ *
+ * @throws NullPointerException
+ * - If the pathname argument is null
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * doesn't allow getting the ProtectionDomain
+ */
+ public static String getCurrentCommandPath() {
+ File f = new File(Slider.class.getProtectionDomain().getCodeSource()
+ .getLocation().getPath());
+ return f.getAbsolutePath();
+ }
+
+ /**
+ * return the path to the slider-client.xml used by the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * denies access to the class loader for the class
+ */
+ public static String getClientConfigPath() {
+ URL path = ConfigHelper.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ return path.toString();
+ }
+
+ /**
+ * validate if slider-client.xml under the path can be opened
+ *
+ * @throws IOException
+ * : the file can't be found or open
+ */
+ public static void validateClientConfigFile() throws IOException {
+ URL resURL = SliderVersionInfo.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ if (resURL == null) {
+ throw new IOException(
+ "slider-client.xml doesn't exist on the path: "
+ + getClientConfigPath());
+ }
+
+ try {
+ InputStream inStream = resURL.openStream();
+ if (inStream == null) {
+ throw new IOException("slider-client.xml can't be opened");
+ }
+ } catch (IOException e) {
+ throw new IOException("slider-client.xml can't be opened: "
+ + e.toString());
+ }
+ }
+
+ /**
+ * validate if a file on HDFS can be open
+ *
+ * @throws IOException the file can't be found or opened
+ * @throws URISyntaxException
+ */
+ public static void validateHDFSFile(SliderFileSystem sliderFileSystem,
+ String pathStr)
+ throws IOException, URISyntaxException {
+ URI pathURI = new URI(pathStr);
+ InputStream inputStream =
+ sliderFileSystem.getFileSystem().open(new Path(pathURI));
+ if (inputStream == null) {
+ throw new IOException("HDFS file " + pathStr + " can't be opened");
+ }
+ }
+
+ /**
+ * return the version and path of the JDK invoking the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String getJDKInfo() {
+ String version = System.getProperty("java.version");
+ String javaHome = System.getProperty("java.home");
+ return
+ "The version of the JDK invoking the current running slider command: "
+ + version + "; The path to it is: " + javaHome;
+ }
+
+ /**
+ * return a description of whether the current user has created credential
+ * cache files from kerberos servers
+ *
+ * @throws IOException
+ * @throws BadConfigException
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String checkCredentialCacheFile() throws IOException,
+ BadConfigException {
+ String result = null;
+ if (!Shell.WINDOWS) {
+ result = Shell.execCommand("klist");
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
index f923ef3..25bb4ab 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceIO.java
@@ -38,15 +38,15 @@ public class InstanceIO {
* Load in an instance definition -but do not resolve it
* @param sliderFileSystem filesystem
* @param clusterDirectory CD
- * @return
+ * @return the unresolved aggregate configuration
* @throws IOException
* @throws SliderException
+ * @throws BadClusterStateException if a lock could not be acquired
*/
public static AggregateConf loadInstanceDefinitionUnresolved(
CoreFileSystem sliderFileSystem,
- Path clusterDirectory) throws
- IOException,
- SliderException {
+ Path clusterDirectory)
+ throws IOException, SliderException {
AggregateConf instanceDefinition = new AggregateConf();
ConfPersister persister =
new ConfPersister(sliderFileSystem, clusterDirectory);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 068b687..6f50fca 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -37,7 +37,6 @@ public class YarnAppListClient {
final String username;
final Configuration conf;
-
public YarnAppListClient(SliderYarnClientImpl yarnClient,
String username,
Configuration conf) {
@@ -76,17 +75,29 @@ public class YarnAppListClient {
public ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
- List<ApplicationReport> instances = listInstances();
+ List<ApplicationReport> instances = listInstances(null);
return yarnClient.findClusterInInstanceList(instances, appname);
}
/**
- * List instances belonging to a specific user
+ * List instances belonging to the specific user
* @return a possibly empty list of AMs
*/
public List<ApplicationReport> listInstances()
- throws YarnException, IOException {
- return yarnClient.listInstances(username);
+ throws YarnException, IOException {
+ return listInstances(null);
+ }
+
+ /**
+ * List instances belonging to a specific user
+ * @return a possibly empty list of AMs
+ * @param user user if not the default. null means default, "" means all users,
+ * otherwise it is the name of a user
+ */
+ public List<ApplicationReport> listInstances(String user)
+ throws YarnException, IOException {
+ String listUser = user == null ? username : user;
+ return yarnClient.listInstances(listUser);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 4b4daf0..1e09d09 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.agent.actions
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
@@ -27,6 +28,7 @@ import org.apache.slider.common.params.ActionListArgs
import org.apache.slider.common.params.ActionThawArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Before
@@ -36,7 +38,6 @@ import org.junit.Test
* Test List operations
*/
@Slf4j
-
class TestActionList extends AgentMiniClusterTestBase {
@Before
@@ -54,10 +55,9 @@ class TestActionList extends AgentMiniClusterTestBase {
@Test
public void testSuite() throws Throwable {
testListThisUserNoClusters()
- testListAllUsersNoClusters()
testListLiveCluster()
testListMissingCluster()
- testActionListHistory()
+ testActionListStates()
}
public void testListThisUserNoClusters() throws Throwable {
@@ -73,28 +73,15 @@ class TestActionList extends AgentMiniClusterTestBase {
)
assert launcher.serviceExitCode == 0
}
-
- public void testListAllUsersNoClusters() throws Throwable {
- log.info("RM address = ${RMAddr}")
- ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
- //config includes RM binding info
- new YarnConfiguration(miniCluster.config),
- //varargs list of command line params
- [
- SliderActions.ACTION_LIST,
- Arguments.ARG_MANAGER, RMAddr,
- ]
- )
- assert launcher.serviceExitCode == 0
- }
public void testListLiveCluster() throws Throwable {
//launch the cluster
- String clustername = createClusterName()
+ String clustername = "testlistlivecluster"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
false)
+
addToTeardown(launcher)
//do the low level operations to get a better view of what is going on
SliderClient sliderClient = launcher.service
@@ -127,7 +114,7 @@ class TestActionList extends AgentMiniClusterTestBase {
SliderActions.ACTION_LIST, clustername
]
)
-
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
}
public void testListMissingCluster() throws Throwable {
@@ -151,8 +138,8 @@ class TestActionList extends AgentMiniClusterTestBase {
}
- public void testActionListHistory() {
- String clustername = createClusterName()
+ public void testActionListStates() {
+ String clustername = "testactionliststates"
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
@@ -161,42 +148,71 @@ class TestActionList extends AgentMiniClusterTestBase {
SliderClient sliderClient = launcher.service
waitForClusterLive(sliderClient)
- ActionListArgs args = new ActionListArgs();
+ describe "listing"
//Listing only live instances
- args.live = true;
- assert sliderClient.actionList(clustername, args) == 0;
+ assert sliderClient.actionList(clustername, new ActionListArgs(live: true)) == 0;
+ assert sliderClient.actionList(clustername,
+ new ActionListArgs(live: true, verbose:true)) == 0;
clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
waitForAppToFinish(sliderClient)
- //Listing only live instances but prints nothing since instance is freezed/stopped
-
- args.live = true;
- args.history = false;
+
try {
- sliderClient.actionList(clustername, args);
- fail("expected a failure")
- } catch (UnknownApplicationInstanceException e) {
-
+ // unknown yarn state
+ int e= sliderClient.actionList(clustername,
+ new ActionListArgs(state: "undefined"));
+ fail("expected failure, got return code of $e")
+ } catch (BadCommandArgumentsException expected) {
+
+ }
+
+ try {
+ // state and --live options
+ int e= sliderClient.actionList(clustername,
+ new ActionListArgs(state: "running", live: true));
+ fail("expected failure, got return code of $e")
+ } catch (BadCommandArgumentsException expected) {
+
}
+ //Listing only live instances but prints nothing since instance is frozen/stopped
+
+ describe("after freeze")
+ // listing finished will work
+ assert 0 == sliderClient.actionList("",
+ new ActionListArgs(state: YarnApplicationState.FINISHED.toString()));
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.FINISHED.toString(),
+ verbose: true));
+
+ assert -1 == sliderClient.actionList("", new ActionListArgs(live: true));
+ assert -1 == sliderClient.actionList(clustername,
+ new ActionListArgs(live: true));
- // historical list will work
- args.history = true;
- assert 0 == sliderClient.actionList(clustername, args)
+ assert -1 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+
+ assert -1 == sliderClient.actionList("",
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
// thaw
- ActionThawArgs thawArgs = new ActionThawArgs();
- sliderClient.actionThaw(clustername, thawArgs);
+ sliderClient.actionThaw(clustername, new ActionThawArgs());
waitForClusterLive(sliderClient)
+ describe("Post-thaw listing")
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
+
//Listing only live instances
- args.live = true;
- args.history = false;
- assert 0 == sliderClient.actionList(clustername, args);
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(live: true));
//Listing all the instance both history (previously freezed instance) and live
args.live = true
- args.history = true
assert 0 == sliderClient.actionList("", args);
+
+ maybeStopCluster(sliderClient, "", "forced", true)
+ assert 0 == sliderClient.actionList(clustername,
+ new ActionListArgs(state: "killed"));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index ed716e0..6593a0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -65,7 +65,7 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
[
SliderActions.ACTION_LIST,
"no-cluster-of-this-name",
- Arguments.ARG_LIVE, Arguments.ARG_HISTORY
+ Arguments.ARG_LIVE
])
fail("expected a failure")
} catch (UnknownApplicationInstanceException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index fb1a980..24bcc0e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -370,14 +370,8 @@ abstract class CommandTestBase extends SliderTestUtils {
slider(cmd)
}
- static SliderShell list(int result, String name) {
- List<String> cmd = [
- ACTION_LIST
- ]
- if (name != null) {
- cmd << name
- }
- slider(result, cmd)
+ static SliderShell list(int result, Collection<String> commands =[]) {
+ slider(result, [ACTION_LIST] + commands )
}
static SliderShell status(String name) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index 20bac88..7ba7fc0 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -30,7 +30,19 @@ public class ListCommandIT extends CommandTestBase {
@Test
public void testListAll() throws Throwable {
- assertSuccess(list(null))
+ assertSuccess(list(""))
+ }
+
+ @Test
+ public void testListAllLive() throws Throwable {
+ def shell = list("--live")
+ assert shell.ret == 0 || shell.ret ==1
+ }
+
+ @Test
+ public void testListAllFinishedLive() throws Throwable {
+ def shell = list("--live")
+ assert shell.ret == 0 || shell.ret == 1
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index dfdbf06..f0fa546 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -92,7 +92,13 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
exists(0, CLUSTER)
//listing the cluster will succeed
- list(0, CLUSTER)
+ list(0, [CLUSTER])
+
+ list(0, [""])
+ list(0, [CLUSTER, ARG_LIVE])
+ list(0, [CLUSTER, ARG_STATE, "running"])
+ list(0, [ARG_LIVE])
+ list(0, [ARG_STATE, "running"])
//simple status
status(0, CLUSTER)
@@ -132,6 +138,15 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
//condition returns false if it is required to be live
exists(EXIT_FALSE, CLUSTER, true)
+ list( 0, [CLUSTER])
+ list( 0, [CLUSTER, ARG_STATE, "FINISHED"])
+ list(-1, [CLUSTER, ARG_LIVE])
+ list(-1, [CLUSTER, ARG_STATE, "running"])
+
+ list(-1, [ARG_LIVE])
+ list(-1, [ARG_STATE, "running"])
+ list( 0, [ARG_STATE, "FINISHED"])
+
//start then stop the cluster
thaw(CLUSTER,
[
@@ -139,6 +154,11 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
])
exists(0, CLUSTER)
describe " >>> Cluster is now thawed."
+ list(0, [CLUSTER, ARG_LIVE])
+ list(0, [CLUSTER, ARG_STATE, "running"])
+ list(0, [ARG_LIVE])
+ list(0, [ARG_STATE, "running"])
+ list(0, [CLUSTER, ARG_STATE, "FINISHED"])
freeze(0, CLUSTER,
[
@@ -180,10 +200,14 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
ARG_MESSAGE, "final-shutdown"
])
+ list(0, [CLUSTER, "--verbose", "--state", "FINISHED"]).dumpOutput()
+
destroy(0, CLUSTER)
//cluster now missing
exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ list(0, [])
+ list(EXIT_UNKNOWN_INSTANCE, [CLUSTER])
} finally {
jsonStatus.delete()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd8384c9/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
index 7943464..5ffee83 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -88,7 +88,7 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
exists(0, CLUSTER)
//listing the cluster will succeed
- list(0, CLUSTER)
+ list(0, [CLUSTER])
//simple status
status(0, CLUSTER)