You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ge...@apache.org on 2014/12/11 22:22:20 UTC
hadoop git commit: HADOOP-11211.
mapreduce.job.classloader.system.classes semantics should be
order-independent. (Yitong Zhou via gera)
Repository: hadoop
Updated Branches:
refs/heads/trunk b9f6d0c95 -> 0bcea111e
HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be order-independent. (Yitong Zhou via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bcea111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bcea111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bcea111
Branch: refs/heads/trunk
Commit: 0bcea111e5daa9a4315346cf6919a4cfc8d90e0d
Parents: b9f6d0c
Author: Gera Shegalov <ge...@apache.org>
Authored: Thu Dec 11 12:25:25 2014 -0800
Committer: Gera Shegalov <ge...@apache.org>
Committed: Thu Dec 11 13:12:13 2014 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/util/ApplicationClassLoader.java | 25 ++++++++++++++++----
.../hadoop/util/TestApplicationClassLoader.java | 8 +++++--
.../src/main/resources/mapred-default.xml | 21 ++++++++++++----
4 files changed, 46 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 47d36e4..d923b87 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -571,6 +571,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11386. Replace \n by %n in format hadoop-common format strings.
(Li Lu via wheat9)
+
+ HADOOP-11211. mapreduce.job.classloader.system.classes semantics should be
+ be order-independent. (Yitong Zhou via gera)
Release 2.6.0 - 2014-11-18
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index d2ab015..9f16b61 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -216,28 +216,43 @@ public class ApplicationClassLoader extends URLClassLoader {
return c;
}
+ /**
+ * Checks if a class should be included as a system class.
+ *
+ * A class is a system class if and only if it matches one of the positive
+ * patterns and none of the negative ones.
+ *
+ * @param name the class name to check
+ * @param systemClasses a list of system class configurations.
+ * @return true if the class is a system class
+ */
public static boolean isSystemClass(String name, List<String> systemClasses) {
+ boolean result = false;
if (systemClasses != null) {
String canonicalName = name.replace('/', '.');
while (canonicalName.startsWith(".")) {
canonicalName=canonicalName.substring(1);
}
for (String c : systemClasses) {
- boolean result = true;
+ boolean shouldInclude = true;
if (c.startsWith("-")) {
c = c.substring(1);
- result = false;
+ shouldInclude = false;
}
if (canonicalName.startsWith(c)) {
if ( c.endsWith(".") // package
|| canonicalName.length() == c.length() // class
|| canonicalName.length() > c.length() // nested
&& canonicalName.charAt(c.length()) == '$' ) {
- return result;
+ if (shouldInclude) {
+ result = true;
+ } else {
+ return false;
+ }
}
}
}
}
- return false;
+ return result;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
index cc16493..be8e61e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
@@ -87,7 +87,7 @@ public class TestApplicationClassLoader {
assertEquals(jarFile.toURI().toURL(), urls[2]);
// nofile should be ignored
}
-
+
@Test
public void testIsSystemClass() {
testIsSystemClassInternal("");
@@ -112,8 +112,12 @@ public class TestApplicationClassLoader {
classes("-org.example.Foo,org.example.")));
assertTrue(isSystemClass("org.example.Bar" + nestedClass,
classes("-org.example.Foo.,org.example.")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.,-org.example.Foo")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.Foo,-org.example.Foo")));
}
-
+
private List<String> classes(String classes) {
return Lists.newArrayList(Splitter.on(',').split(classes));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bcea111/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 00a89c9..6e0deaa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1363,10 +1363,23 @@
<value></value>
<description>Used to override the default definition of the system classes for
the job classloader. The system classes are a comma-separated list of
- classes that should be loaded from the system classpath, not the
- user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending
- in '.' (period) are treated as package names, and names starting with a '-'
- are treated as negative matches.
+ patterns that indicate whether to load a class from the system classpath,
+ instead from the user-supplied JARs, when mapreduce.job.classloader is
+ enabled.
+
+ A positive pattern is defined as:
+ 1. A single class name 'C' that matches 'C' and transitively all nested
+ classes 'C$*' defined in C;
+ 2. A package name ending with a '.' (e.g., "com.example.") that matches
+ all classes from that package.
+ A negative pattern is defined by a '-' in front of a positive pattern
+ (e.g., "-com.example.").
+
+ A class is considered a system class if and only if it matches one of the
+ positive patterns and none of the negative ones. More formally:
+ A class is a member of the inclusion set I if it matches one of the positive
+ patterns. A class is a member of the exclusion set E if it matches one of
+ the negative patterns. The set of system classes S = I \ E.
</description>
</property>