You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/07/06 23:51:00 UTC
hadoop git commit: YARN-5233. Support for specifying a path for ATS
plugin jars. Contributed by Li Lu
Repository: hadoop
Updated Branches:
refs/heads/trunk 4c9e1aeb9 -> 8a9d293dd
YARN-5233. Support for specifying a path for ATS plugin jars. Contributed by Li Lu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a9d293d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a9d293d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a9d293d
Branch: refs/heads/trunk
Commit: 8a9d293dd60f6d51e1574e412d40746ba8175fe1
Parents: 4c9e1ae
Author: Jian He <ji...@apache.org>
Authored: Wed Jul 6 16:44:46 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Wed Jul 6 16:44:46 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 ++
.../src/main/resources/yarn-default.xml | 9 +++
.../timeline/EntityGroupFSTimelineStore.java | 77 +++++++++++++++++---
.../TestEntityGroupFSTimelineStore.java | 46 ++++++++++++
4 files changed, 131 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9d293d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 0beec62..c0241c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1824,6 +1824,14 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES =
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "group-id-plugin-classes";
+ public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH =
+ TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
+ + "group-id-plugin-classpath";
+
+ public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES
+ = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
+ + "group-id-plugin-system-classes";
+
public static final String
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE =
TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "summary-store";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9d293d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index eabb679..cb0ac07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2084,6 +2084,15 @@
</description>
</property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath</name>
+ <value></value>
+ <description>
+ Classpath for all plugins defined in
+ yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes.
+ </description>
+ </property>
+
<property>
<name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
<description>Summary storage for ATS v1.5</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9d293d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
index 958b54e..7e6c154 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.ServiceOperations;
import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.util.ApplicationClassLoader;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.MappingJsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
@@ -58,7 +58,12 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.MalformedURLException;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -217,25 +222,47 @@ public class EntityGroupFSTimelineStore extends CompositeService
throws RuntimeException {
Collection<String> pluginNames = conf.getTrimmedStringCollection(
YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES);
+
+ String pluginClasspath = conf.getTrimmed(
+ YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH);
+ String[] systemClasses = conf.getTrimmedStrings(
+ YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES);
+
List<TimelineEntityGroupPlugin> pluginList
= new LinkedList<TimelineEntityGroupPlugin>();
Exception caught = null;
+ ClassLoader customClassLoader = null;
+ if (pluginClasspath != null && pluginClasspath.length() > 0) {
+ try {
+ customClassLoader = createPluginClassLoader(pluginClasspath,
+ systemClasses);
+ } catch (IOException ioe) {
+ LOG.warn("Error loading classloader", ioe);
+ }
+ }
for (final String name : pluginNames) {
LOG.debug("Trying to load plugin class {}", name);
TimelineEntityGroupPlugin cacheIdPlugin = null;
+
try {
- Class<?> clazz = conf.getClassByName(name);
- cacheIdPlugin =
- (TimelineEntityGroupPlugin) ReflectionUtils.newInstance(
- clazz, conf);
+ if (customClassLoader != null) {
+ LOG.debug("Load plugin {} with classpath: {}", name, pluginClasspath);
+ Class<?> clazz = Class.forName(name, true, customClassLoader);
+ Class<? extends TimelineEntityGroupPlugin> sClass = clazz.asSubclass(
+ TimelineEntityGroupPlugin.class);
+ cacheIdPlugin = ReflectionUtils.newInstance(sClass, conf);
+ } else {
+ LOG.debug("Load plugin class with system classpath");
+ Class<?> clazz = conf.getClassByName(name);
+ cacheIdPlugin =
+ (TimelineEntityGroupPlugin) ReflectionUtils.newInstance(
+ clazz, conf);
+ }
} catch (Exception e) {
LOG.warn("Error loading plugin " + name, e);
- caught = e;
+ throw new RuntimeException("No class defined for " + name, e);
}
- if (cacheIdPlugin == null) {
- throw new RuntimeException("No class defined for " + name, caught);
- }
LOG.info("Load plugin class {}", cacheIdPlugin.getClass().getName());
pluginList.add(cacheIdPlugin);
}
@@ -489,6 +516,29 @@ public class EntityGroupFSTimelineStore extends CompositeService
return appId;
}
+ private static ClassLoader createPluginClassLoader(
+ final String appClasspath, final String[] systemClasses)
+ throws IOException {
+ try {
+ return AccessController.doPrivileged(
+ new PrivilegedExceptionAction<ClassLoader>() {
+ @Override
+ public ClassLoader run() throws MalformedURLException {
+ return new ApplicationClassLoader(appClasspath,
+ EntityGroupFSTimelineStore.class.getClassLoader(),
+ Arrays.asList(systemClasses));
+ }
+ }
+ );
+ } catch (PrivilegedActionException e) {
+ Throwable t = e.getCause();
+ if (t instanceof MalformedURLException) {
+ throw (MalformedURLException) t;
+ }
+ throw new IOException(e);
+ }
+ }
+
private Path getActiveAppPath(ApplicationId appId) {
return new Path(activeRootPath, appId.toString());
}
@@ -532,6 +582,15 @@ public class EntityGroupFSTimelineStore extends CompositeService
}
/**
+ * Get all plugins for tests.
+ * @return all plugins
+ */
+ @VisibleForTesting
+ List<TimelineEntityGroupPlugin> getPlugins() {
+ return cacheIdPlugins;
+ }
+
+ /**
* Ask the RM for the state of the application.
* This method has to be synchronized to control traffic to RM
* @param appId application ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9d293d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
index 1c12f36..95d0987 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestEntityGroupFSTimelineStore.java
@@ -29,6 +29,9 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.metrics2.lib.MutableCounterLong;
import org.apache.hadoop.metrics2.lib.MutableStat;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ApplicationClassLoader;
+import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -43,7 +46,10 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import java.io.File;
import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
@@ -97,6 +103,8 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
@Rule
public TestName currTestName = new TestName();
+ private File rootDir;
+ private File testJar;
@BeforeClass
public static void setupClass() throws Exception {
@@ -144,6 +152,25 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
store = new EntityGroupFSTimelineStoreForTest();
if (currTestName.getMethodName().contains("Plugin")) {
+ rootDir = GenericTestUtils.getTestDir(getClass()
+ .getSimpleName());
+ if (!rootDir.exists()) {
+ rootDir.mkdirs();
+ }
+ testJar = null;
+ testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir,
+ "test-runjar.jar", 2048,
+ EntityGroupPlugInForTest.class.getName());
+ config.set(
+ YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH,
+ testJar.getAbsolutePath());
+ // add "-org.apache.hadoop." as system classes
+ String systemClasses = "-org.apache.hadoop." + "," +
+ ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT;
+ config.set(
+ YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES,
+ systemClasses);
+
config.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES,
EntityGroupPlugInForTest.class.getName());
}
@@ -158,6 +185,10 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
for (ApplicationId appId : sampleAppIds) {
fs.delete(getTestRootPath(appId.toString()), true);
}
+ if (testJar != null) {
+ testJar.delete();
+ rootDir.delete();
+ }
}
@AfterClass
@@ -302,6 +333,21 @@ public class TestEntityGroupFSTimelineStore extends TimelineStoreTestUtils {
assertEquals(EntityGroupPlugInForTest.class.getName(),
store.getConfig().get(
YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES));
+ List<TimelineEntityGroupPlugin> currPlugins = store.getPlugins();
+ for (TimelineEntityGroupPlugin plugin : currPlugins) {
+ ClassLoader pluginClassLoader = plugin.getClass().getClassLoader();
+ assertTrue("Should set up ApplicationClassLoader",
+ pluginClassLoader instanceof ApplicationClassLoader);
+ URL[] paths = ((URLClassLoader) pluginClassLoader).getURLs();
+ boolean foundJAR = false;
+ for (URL path : paths) {
+ if (path.toString().contains(testJar.getAbsolutePath())) {
+ foundJAR = true;
+ }
+ }
+ assertTrue("Not found path " + testJar.getAbsolutePath()
+ + " for plugin " + plugin.getClass().getName(), foundJAR);
+ }
// Load data and cache item, prepare timeline store by making a cache item
EntityGroupFSTimelineStore.AppLogs appLogs =
store.new AppLogs(mainTestAppId, mainTestAppDirPath,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org