You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cu...@apache.org on 2014/08/21 23:56:08 UTC
svn commit: r1619608 - in
/hadoop/common/branches/YARN-1051/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache...
Author: curino
Date: Thu Aug 21 21:55:57 2014
New Revision: 1619608
URL: http://svn.apache.org/r1619608
Log:
Merge with trunk to pick up YARN-2436
Added:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
- copied unchanged from r1619607, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
Removed:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt Thu Aug 21 21:55:57 2014
@@ -18,6 +18,8 @@ Trunk - Unreleased
YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk.
(Zhijie Shen via xgong)
+ YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -50,6 +52,9 @@ Release 2.6.0 - UNRELEASED
YARN-2411. Support simple user and group mappings to queues. (Ram Venkatesh
via jianhe)
+ YARN-2174. Enable HTTPs for the writer REST API of TimelineServer.
+ (Zhijie Shen via jianhe)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
@@ -217,7 +222,16 @@ Release 2.6.0 - UNRELEASED
YARN-2249. Avoided AM release requests being lost on work preserving RM
restart. (Jian He via zjshen)
-Release 2.5.0 - UNRELEASED
+ YARN-2034. Description for yarn.nodemanager.localizer.cache.target-size-mb
+ is incorrect (Chen He via jlowe)
+
+ YARN-1919. Potential NPE in EmbeddedElectorService#stop.
+ (Tsuyoshi Ozawa via kasha)
+
+ YARN-2424. LCE should support non-cgroups, non-secure mode (Chris Douglas
+ via aw)
+
+Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn Thu Aug 21 21:55:57 2014
@@ -73,6 +73,7 @@ case "${COMMAND}" in
application|applicationattempt|container)
CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ set -- "${COMMAND}" "$@"
;;
classpath)
hadoop_finalize
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Thu Aug 21 21:55:57 2014
@@ -344,4 +344,11 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider"/>
<Bug pattern="DC_DOUBLECHECK" />
</Match>
+
+ <!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
+ warning on the identical name as it should be removed later -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.ApplicationClassLoader"/>
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Aug 21 21:55:57 2014
@@ -606,7 +606,11 @@ public class YarnConfiguration extends C
public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
10 * 60 * 1000;
- /** Target size of localizer cache in MB, per local directory.*/
+ /**
+ * Target size of localizer cache in MB, per nodemanager. It is a target
+ * retention size that only includes resources with PUBLIC and PRIVATE
+ * visibility and excludes resources with APPLICATION visibility
+ */
public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
NM_PREFIX + "localizer.cache.target-size-mb";
public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;
@@ -833,6 +837,15 @@ public class YarnConfiguration extends C
NM_PREFIX + "linux-container-executor.group";
/**
+ * True if linux-container-executor should limit itself to one user
+ * when running in non-secure mode.
+ */
+ public static final String NM_NONSECURE_MODE_LIMIT_USERS = NM_PREFIX +
+ "linux-container-executor.nonsecure-mode.limit-users";
+
+ public static final boolean DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS = true;
+
+ /**
* The UNIX user that containers will run as when Linux-container-executor
* is used in nonsecure mode (a use case for this is using cgroups).
*/
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java Thu Aug 21 21:55:57 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.Authenticator;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDelegationTokenResponse;
@@ -53,10 +54,13 @@ import com.google.common.annotations.Vis
public class TimelineAuthenticator extends KerberosAuthenticator {
private static ObjectMapper mapper;
+ private static TimelineAuthenticator authenticator;
+ private static ConnectionConfigurator connConfigurator;
static {
mapper = new ObjectMapper();
YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+ authenticator = new TimelineAuthenticator();
}
/**
@@ -98,6 +102,11 @@ public class TimelineAuthenticator exten
}
}
+ public static void setStaticConnectionConfigurator(
+ ConnectionConfigurator connConfigurator) {
+ TimelineAuthenticator.connConfigurator = connConfigurator;
+ }
+
public static Token<TimelineDelegationTokenIdentifier> getDelegationToken(
URL url, AuthenticatedURL.Token token, String renewer) throws IOException {
TimelineDelegationTokenOperation op =
@@ -107,7 +116,7 @@ public class TimelineAuthenticator exten
params.put(TimelineAuthenticationConsts.RENEWER_PARAM, renewer);
url = appendParams(url, params);
AuthenticatedURL aUrl =
- new AuthenticatedURL(new TimelineAuthenticator());
+ new AuthenticatedURL(authenticator, connConfigurator);
try {
HttpURLConnection conn = aUrl.openConnection(url, token);
conn.setRequestMethod(op.getHttpMethod());
@@ -137,7 +146,7 @@ public class TimelineAuthenticator exten
dToken.encodeToUrlString());
url = appendParams(url, params);
AuthenticatedURL aUrl =
- new AuthenticatedURL(new TimelineAuthenticator());
+ new AuthenticatedURL(authenticator, connConfigurator);
try {
HttpURLConnection conn = aUrl.openConnection(url, token);
conn.setRequestMethod(
@@ -164,7 +173,7 @@ public class TimelineAuthenticator exten
dToken.encodeToUrlString());
url = appendParams(url, params);
AuthenticatedURL aUrl =
- new AuthenticatedURL(new TimelineAuthenticator());
+ new AuthenticatedURL(authenticator, connConfigurator);
try {
HttpURLConnection conn = aUrl.openConnection(url, token);
conn.setRequestMethod(TimelineDelegationTokenOperation.CANCELDELEGATIONTOKEN
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java Thu Aug 21 21:55:57 2014
@@ -23,10 +23,15 @@ import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.net.URLConnection;
+import java.security.GeneralSecurityException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
import javax.ws.rs.core.MediaType;
import org.apache.commons.cli.CommandLine;
@@ -42,6 +47,8 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
@@ -74,7 +81,10 @@ public class TimelineClientImpl extends
private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
private static final String URL_PARAM_USER_NAME = "user.name";
private static final Joiner JOINER = Joiner.on("");
+ public final static int DEFAULT_SOCKET_TIMEOUT = 1 * 60 * 1000; // 1 minute
+
private static Options opts;
+
static {
opts = new Options();
opts.addOption("put", true, "Put the TimelineEntities in a JSON file");
@@ -89,15 +99,6 @@ public class TimelineClientImpl extends
public TimelineClientImpl() {
super(TimelineClientImpl.class.getName());
- ClientConfig cc = new DefaultClientConfig();
- cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
- if (UserGroupInformation.isSecurityEnabled()) {
- urlFactory = new KerberosAuthenticatedURLConnectionFactory();
- client = new Client(new URLConnectionClientHandler(urlFactory), cc);
- } else {
- client = new Client(new URLConnectionClientHandler(
- new PseudoAuthenticatedURLConnectionFactory()), cc);
- }
}
protected void serviceInit(Configuration conf) throws Exception {
@@ -107,6 +108,17 @@ public class TimelineClientImpl extends
if (!isEnabled) {
LOG.info("Timeline service is not enabled");
} else {
+ ClientConfig cc = new DefaultClientConfig();
+ cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ ConnectionConfigurator connConfigurator = newConnConfigurator(conf);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ TimelineAuthenticator.setStaticConnectionConfigurator(connConfigurator);
+ urlFactory = new KerberosAuthenticatedURLConnectionFactory(connConfigurator);
+ client = new Client(new URLConnectionClientHandler(urlFactory), cc);
+ } else {
+ client = new Client(new URLConnectionClientHandler(
+ new PseudoAuthenticatedURLConnectionFactory(connConfigurator)), cc);
+ }
if (YarnConfiguration.useHttps(conf)) {
resURI = URI
.create(JOINER.join("https://", conf.get(
@@ -182,6 +194,13 @@ public class TimelineClientImpl extends
private static class PseudoAuthenticatedURLConnectionFactory
implements HttpURLConnectionFactory {
+ private ConnectionConfigurator connConfigurator;
+
+ public PseudoAuthenticatedURLConnectionFactory(
+ ConnectionConfigurator connConfigurator) {
+ this.connConfigurator = connConfigurator;
+ }
+
@Override
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
Map<String, String> params = new HashMap<String, String>();
@@ -191,7 +210,7 @@ public class TimelineClientImpl extends
if (LOG.isDebugEnabled()) {
LOG.debug("URL with delegation token: " + url);
}
- return (HttpURLConnection) url.openConnection();
+ return connConfigurator.configure((HttpURLConnection) url.openConnection());
}
}
@@ -202,10 +221,13 @@ public class TimelineClientImpl extends
private TimelineAuthenticator authenticator;
private Token<TimelineDelegationTokenIdentifier> dToken;
private Text service;
+ private ConnectionConfigurator connConfigurator;
- public KerberosAuthenticatedURLConnectionFactory() {
+ public KerberosAuthenticatedURLConnectionFactory(
+ ConnectionConfigurator connConfigurator) {
token = new AuthenticatedURL.Token();
authenticator = new TimelineAuthenticator();
+ this.connConfigurator = connConfigurator;
}
@Override
@@ -226,7 +248,8 @@ public class TimelineClientImpl extends
LOG.debug("URL with delegation token: " + url);
}
}
- return new AuthenticatedURL(authenticator).openConnection(url, token);
+ return new AuthenticatedURL(
+ authenticator, connConfigurator).openConnection(url, token);
} catch (AuthenticationException e) {
LOG.error("Authentication failed when openning connection [" + url
+ "] with token [" + token + "].", e);
@@ -255,6 +278,57 @@ public class TimelineClientImpl extends
}
+ private static ConnectionConfigurator newConnConfigurator(Configuration conf) {
+ try {
+ return newSslConnConfigurator(DEFAULT_SOCKET_TIMEOUT, conf);
+ } catch (Exception e) {
+ LOG.debug("Cannot load customized ssl related configuration. " +
+ "Fallback to system-generic settings.", e);
+ return DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
+ }
+ }
+
+ private static final ConnectionConfigurator DEFAULT_TIMEOUT_CONN_CONFIGURATOR =
+ new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ setTimeouts(conn, DEFAULT_SOCKET_TIMEOUT);
+ return conn;
+ }
+ };
+
+ private static ConnectionConfigurator newSslConnConfigurator(final int timeout,
+ Configuration conf) throws IOException, GeneralSecurityException {
+ final SSLFactory factory;
+ final SSLSocketFactory sf;
+ final HostnameVerifier hv;
+
+ factory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ factory.init();
+ sf = factory.createSSLSocketFactory();
+ hv = factory.getHostnameVerifier();
+
+ return new ConnectionConfigurator() {
+ @Override
+ public HttpURLConnection configure(HttpURLConnection conn)
+ throws IOException {
+ if (conn instanceof HttpsURLConnection) {
+ HttpsURLConnection c = (HttpsURLConnection) conn;
+ c.setSSLSocketFactory(sf);
+ c.setHostnameVerifier(hv);
+ }
+ setTimeouts(conn, timeout);
+ return conn;
+ }
+ };
+ }
+
+ private static void setTimeouts(URLConnection connection, int socketTimeout) {
+ connection.setConnectTimeout(socketTimeout);
+ connection.setReadTimeout(socketTimeout);
+ }
+
public static void main(String[] argv) throws Exception {
CommandLine cliParser = new GnuParser().parse(opts, argv);
if (cliParser.hasOption("put")) {
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java Thu Aug 21 21:55:57 2014
@@ -18,180 +18,30 @@
package org.apache.hadoop.yarn.util;
-import java.io.File;
-import java.io.FilenameFilter;
import java.net.MalformedURLException;
import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-
/**
- * A {@link URLClassLoader} for YARN application isolation. Classes from
- * the application JARs are loaded in preference to the parent loader.
+ * This type has been deprecated in favor of
+ * {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ * ApplicationClassLoader should use that type instead.
*/
@Public
@Unstable
-public class ApplicationClassLoader extends URLClassLoader {
-
- private static final Log LOG =
- LogFactory.getLog(ApplicationClassLoader.class.getName());
-
- private static final FilenameFilter JAR_FILENAME_FILTER =
- new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(".jar") || name.endsWith(".JAR");
- }
- };
-
- private ClassLoader parent;
- private List<String> systemClasses;
-
+@Deprecated
+public class ApplicationClassLoader extends
+ org.apache.hadoop.util.ApplicationClassLoader {
public ApplicationClassLoader(URL[] urls, ClassLoader parent,
List<String> systemClasses) {
- super(urls, parent);
- this.parent = parent;
- if (parent == null) {
- throw new IllegalArgumentException("No parent classloader!");
- }
- this.systemClasses = systemClasses;
+ super(urls, parent, systemClasses);
}
-
+
public ApplicationClassLoader(String classpath, ClassLoader parent,
List<String> systemClasses) throws MalformedURLException {
- this(constructUrlsFromClasspath(classpath), parent, systemClasses);
- }
-
- @VisibleForTesting
- static URL[] constructUrlsFromClasspath(String classpath)
- throws MalformedURLException {
- List<URL> urls = new ArrayList<URL>();
- for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
- if (element.endsWith("/*")) {
- String dir = element.substring(0, element.length() - 1);
- File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
- if (files != null) {
- for (File file : files) {
- urls.add(file.toURI().toURL());
- }
- }
- } else {
- File file = new File(element);
- if (file.exists()) {
- urls.add(new File(element).toURI().toURL());
- }
- }
- }
- return urls.toArray(new URL[urls.size()]);
- }
-
- @Override
- public URL getResource(String name) {
- URL url = null;
-
- if (!isSystemClass(name, systemClasses)) {
- url= findResource(name);
- if (url == null && name.startsWith("/")) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Remove leading / off " + name);
- }
- url= findResource(name.substring(1));
- }
- }
-
- if (url == null) {
- url= parent.getResource(name);
- }
-
- if (url != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("getResource("+name+")=" + url);
- }
- }
-
- return url;
- }
-
- @Override
- public Class<?> loadClass(String name) throws ClassNotFoundException {
- return this.loadClass(name, false);
- }
-
- @Override
- protected synchronized Class<?> loadClass(String name, boolean resolve)
- throws ClassNotFoundException {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading class: " + name);
- }
-
- Class<?> c = findLoadedClass(name);
- ClassNotFoundException ex = null;
-
- if (c == null && !isSystemClass(name, systemClasses)) {
- // Try to load class from this classloader's URLs. Note that this is like
- // the servlet spec, not the usual Java 2 behaviour where we ask the
- // parent to attempt to load first.
- try {
- c = findClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class: " + name + " ");
- }
- } catch (ClassNotFoundException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(e);
- }
- ex = e;
- }
- }
-
- if (c == null) { // try parent
- c = parent.loadClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class from parent: " + name + " ");
- }
- }
-
- if (c == null) {
- throw ex != null ? ex : new ClassNotFoundException(name);
- }
-
- if (resolve) {
- resolveClass(c);
- }
-
- return c;
- }
-
- @VisibleForTesting
- public static boolean isSystemClass(String name, List<String> systemClasses) {
- if (systemClasses != null) {
- String canonicalName = name.replace('/', '.');
- while (canonicalName.startsWith(".")) {
- canonicalName=canonicalName.substring(1);
- }
- for (String c : systemClasses) {
- boolean result = true;
- if (c.startsWith("-")) {
- c = c.substring(1);
- result = false;
- }
- if (c.endsWith(".") && canonicalName.startsWith(c)) {
- return result;
- } else if (canonicalName.equals(c)) {
- return result;
- }
- }
- }
- return false;
+ super(classpath, parent, systemClasses);
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Thu Aug 21 21:55:57 2014
@@ -757,7 +757,10 @@
</property>
<property>
- <description>Target size of localizer cache in MB, per local directory.</description>
+ <description>Target size of localizer cache in MB, per nodemanager. It is
+ a target retention size that only includes resources with PUBLIC and
+ PRIVATE visibility and excludes resources with APPLICATION visibility
+ </description>
<name>yarn.nodemanager.localizer.cache.target-size-mb</name>
<value>10240</value>
</property>
@@ -988,8 +991,22 @@
</property>
<property>
- <description>The UNIX user that containers will run as when Linux-container-executor
- is used in nonsecure mode (a use case for this is using cgroups).</description>
+ <description>This determines which of the two modes that LCE should use on
+ a non-secure cluster. If this value is set to true, then all containers
+ will be launched as the user specified in
+ yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user. If
+ this value is set to false, then containers will run as the user who
+ submitted the application.</description>
+ <name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>The UNIX user that containers will run as when
+ Linux-container-executor is used in nonsecure mode (a use case for this
+ is using cgroups) if the
+ yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is
+ set to true.</description>
<name>yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user</name>
<value>nobody</value>
</property>
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Thu Aug 21 21:55:57 2014
@@ -57,8 +57,8 @@ public class LinuxContainerExecutor exte
private LCEResourcesHandler resourcesHandler;
private boolean containerSchedPriorityIsSet = false;
private int containerSchedPriorityAdjustment = 0;
-
-
+ private boolean containerLimitUsers;
+
@Override
public void setConf(Configuration conf) {
super.setConf(conf);
@@ -81,6 +81,13 @@ public class LinuxContainerExecutor exte
nonsecureLocalUserPattern = Pattern.compile(
conf.get(YarnConfiguration.NM_NONSECURE_MODE_USER_PATTERN_KEY,
YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_USER_PATTERN));
+ containerLimitUsers = conf.getBoolean(
+ YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS,
+ YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS);
+ if (!containerLimitUsers) {
+ LOG.warn(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS +
+ ": impersonation without authentication enabled");
+ }
}
void verifyUsernamePattern(String user) {
@@ -92,7 +99,12 @@ public class LinuxContainerExecutor exte
}
String getRunAsUser(String user) {
- return UserGroupInformation.isSecurityEnabled() ? user : nonsecureLocalUser;
+ if (UserGroupInformation.isSecurityEnabled() ||
+ !containerLimitUsers) {
+ return user;
+ } else {
+ return nonsecureLocalUser;
+ }
}
/**
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java Thu Aug 21 21:55:57 2014
@@ -279,6 +279,13 @@ public class TestLinuxContainerExecutor
lce.setConf(conf);
Assert.assertEquals("bar", lce.getRunAsUser("foo"));
+ //nonsecure without limits
+ conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, "bar");
+ conf.setBoolean(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS, false);
+ lce = new LinuxContainerExecutor();
+ lce.setConf(conf);
+ Assert.assertEquals("foo", lce.getRunAsUser("foo"));
+
//secure
conf = new YarnConfiguration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1619608&r1=1619607&r2=1619608&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java Thu Aug 21 21:55:57 2014
@@ -109,8 +109,14 @@ public class EmbeddedElectorService exte
@Override
protected void serviceStop() throws Exception {
- elector.quitElection(false);
- elector.terminateConnection();
+ /**
+ * When error occurs in serviceInit(), serviceStop() can be called.
+ * We need null check for the case.
+ */
+ if (elector != null) {
+ elector.quitElection(false);
+ elector.terminateConnection();
+ }
super.serviceStop();
}