You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/13 22:53:48 UTC
[56/74] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate
expensive YARN get all applications call. Contributed by Gour Saha
YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdea5e1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdea5e1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdea5e1d
Branch: refs/heads/yarn-native-services
Commit: fdea5e1d2b1d1967540570486fffb081ee032ea0
Parents: 038561c
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Dec 13 14:46:44 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 +-
.../slider/client/SliderYarnClientImpl.java | 94 +++++++++++++-------
.../apache/slider/common/tools/SliderUtils.java | 37 +++++++-
.../slider/core/registry/YarnAppListClient.java | 21 ++++-
.../providers/AbstractClientProvider.java | 28 +++++-
.../providers/docker/DockerClientProvider.java | 8 ++
.../servicemonitor/YarnApplicationProbe.java | 12 +--
7 files changed, 160 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
- appOperations);
+ appOperations, clustername);
Credentials credentials = null;
if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
} else {
// scan for instance in single --state state
- List<ApplicationReport> userInstances = yarnClient.listDeployedInstances("");
state = state.toUpperCase(Locale.ENGLISH);
YarnApplicationState desiredState = extractYarnApplicationState(state);
+ List<ApplicationReport> userInstances = yarnClient
+ .listDeployedInstances("", EnumSet.of(desiredState), name);
ApplicationReport foundInstance =
yarnClient.findAppInInstanceList(userInstances, name, desiredState);
if (foundInstance != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
+ return listDeployedInstances(user, null);
+ }
+
+ /**
+ * List Slider <i>deployed</i>instances belonging to a specific user in a
+ * given set of states.
+ * <p>
+ * Deployed means: known about in the YARN cluster; it will include all apps
+ * in the specified set of states.
+ *
+ * @param user
+ * user: "" means all users
+ * @param appStates
+ * filter by a set of YarnApplicationState
+ * @return a possibly empty list of Slider AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listDeployedInstances(String user,
+ EnumSet<YarnApplicationState> appStates)
+ throws YarnException, IOException {
+ return listDeployedInstances(user, appStates, null);
+ }
+
+ /**
+ * List Slider <i>deployed</i>instances belonging to a specific user in a
+ * given set of states and filtered by an application name tag.
+ * <p>
+ * Deployed means: known about in the YARN cluster; it will include all apps
+ * in the specified set of states and tagged with the specified app name.
+ *
+ * @param user
+ * user: "" means all users
+ * @param appStates
+ * filter by a set of YarnApplicationState
+ * @param appname
+ * an application name tag in the format defined by
+ * {@link SliderUtils#createNameTag(String)}
+ * @return a possibly empty list of Slider AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listDeployedInstances(String user,
+ EnumSet<YarnApplicationState> appStates, String appname)
+ throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
- List<ApplicationReport> allApps = getApplications(types);
+ Set<String> tags = null;
+ if (appname != null) {
+ tags = Collections.singleton(SliderUtils.createNameTag(appname));
+ }
+ List<ApplicationReport> allApps = getApplications(types, appStates, tags);
List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
@@ -136,20 +187,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
* @param appname application name
* @return the list of all matching application instances
*/
- public List<ApplicationReport> findAllInstances(String user,
- String appname)
+ public List<ApplicationReport> findAllInstances(String user, String appname)
throws IOException, YarnException {
Preconditions.checkArgument(appname != null, "Null application name");
- List<ApplicationReport> instances = listDeployedInstances(user);
- List<ApplicationReport> results =
- new ArrayList<>(instances.size());
- for (ApplicationReport report : instances) {
- if (report.getName().equals(appname)) {
- results.add(report);
- }
- }
- return results;
+ return listDeployedInstances(user, null, appname);
}
/**
@@ -204,14 +246,12 @@ public class SliderYarnClientImpl extends YarnClientImpl {
// user wants all instances killed
String user = getUsername();
log.info("Killing all applications belonging to {}", user);
- Collection<ApplicationReport> instances = listDeployedInstances(user);
+ Collection<ApplicationReport> instances = listDeployedInstances(user,
+ SliderUtils.getAllLiveAppStates());
for (ApplicationReport instance : instances) {
- if (isApplicationLive(instance)) {
- ApplicationId appId = instance.getApplicationId();
- log.info("Killing Application {}", appId);
-
- killRunningApplication(appId, "forced kill");
- }
+ ApplicationId appId = instance.getApplicationId();
+ log.info("Killing Application {}", appId);
+ killRunningApplication(appId, "forced kill");
}
} else {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
@@ -290,21 +330,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllLiveInstances(String user,
- String appname) throws
- YarnException,
- IOException {
+ String appname) throws YarnException, IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
- List<ApplicationReport> instances = listDeployedInstances(user);
- List<ApplicationReport> results =
- new ArrayList<ApplicationReport>(instances.size());
- for (ApplicationReport app : instances) {
- if (app.getName().equals(appname)
- && isApplicationLive(app)) {
- results.add(app);
- }
- }
- return results;
+ return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(),
+ appname);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b101d34..4457957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -104,6 +104,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
@@ -2656,4 +2657,38 @@ public final class SliderUtils {
}
return prefix;
}
-}
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ /**
+ * Get all YarnApplicationState values which signify that an application is
+ * in RUNNING or pre-RUNNING state.
+ *
+ * @return all live app states
+ */
+ public static EnumSet<YarnApplicationState> getAllLiveAppStates() {
+ return EnumSet.range(YarnApplicationState.NEW,
+ YarnApplicationState.RUNNING);
+ }
+
+ /**
+ * Get all YarnApplicationState values which signify that an application is
+ * not live, which means it is in one of the post RUNNING states.
+ *
+ * @return all non live app states
+ */
+ public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() {
+ return EnumSet.range(YarnApplicationState.FINISHED,
+ YarnApplicationState.KILLED);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 1bdfb9c..71cc193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -86,7 +86,7 @@ public class YarnAppListClient {
public ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
- List<ApplicationReport> instances = listInstances(null);
+ List<ApplicationReport> instances = listInstances(null, appname);
return yarnClient.findClusterInInstanceList(instances, appname);
}
@@ -107,8 +107,25 @@ public class YarnAppListClient {
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
+ return listInstances(user, null);
+ }
+
+ /**
+ * List all instances belonging to a specific user and a specific appname.
+ *
+ * @param user
+ * user if not the default. null means default, "" means all users,
+ * otherwise it is the name of a user
+ * @param appname
+ * application name set as an application tag
+ * @return a possibly empty list of AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listInstances(String user, String appname)
+ throws YarnException, IOException {
String listUser = user == null ? username : user;
- return yarnClient.listDeployedInstances(listUser);
+ return yarnClient.listDeployedInstances(listUser, null, appname);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index f59c347..01444fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
@@ -37,6 +38,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import java.util.HashSet;
import java.util.Set;
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
@@ -217,11 +219,35 @@ public abstract class AbstractClientProvider extends Configured {
* @return the set of tags.
*/
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
- ConfTreeOperations appConf) throws SliderException {
+ ConfTreeOperations appConf, String appName) throws SliderException {
return Collections.emptySet();
}
/**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
* Process client operations for applications such as install, configure
* @param fileSystem
* @param registryOperations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index d554427..86d87ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -34,6 +34,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
@@ -93,4 +94,11 @@ public class DockerClientProvider extends AbstractClientProvider
}
}
}
+
+ @Override
+ public Set<String> getApplicationTags(SliderFileSystem fileSystem,
+ ConfTreeOperations appConf, String appName) throws SliderException {
+ return createApplicationTags(appName, null, null);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
index adf613c..92df048 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
@@ -68,23 +68,19 @@ public class YarnApplicationProbe extends Probe {
*/
@Override
public ProbeStatus ping(boolean livePing) {
-
ProbeStatus status = new ProbeStatus();
try {
-
- List<ApplicationReport> instances =
- yarnClient.listDeployedInstances(username);
- ApplicationReport instance =
- yarnClient.findClusterInInstanceList(instances, clustername);
+ List<ApplicationReport> instances = yarnClient
+ .listDeployedInstances(username, null, clustername);
+ ApplicationReport instance = yarnClient
+ .findClusterInInstanceList(instances, clustername);
if (null == instance) {
throw UnknownApplicationInstanceException.unknownInstance(clustername);
}
-
status.succeed(this);
} catch (Exception e) {
status.fail(this, e);
}
return status;
-
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org