You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2015/12/19 03:41:51 UTC
[1/3] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure
init for unused FileSystems. Contributed by Inigo Goiri
Repository: hadoop
Updated Branches:
refs/heads/branch-2 dd91cf9fa -> d984ba4fd
refs/heads/branch-2.8 d6f1d3b6c -> 4150d08a4
refs/heads/trunk 932956458 -> 8652cce5b
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8652cce5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8652cce5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8652cce5
Branch: refs/heads/trunk
Commit: 8652cce5b21825f6f835e4ea31de82eb59fb06c1
Parents: 9329564
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Dec 18 18:30:18 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++--
.../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++--
.../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++
3 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8652cce5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a79c85d..b0bcf91 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1412,8 +1412,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-12656. MiniKdc throws "address in use" BindException.
(Wei-Chiu Chuang via Arpit Agarwal)
- OPTIMIZATIONS
-
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
over getMessage() in logging/span events. (Varun Saxena via stevel)
@@ -1526,6 +1524,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
server. (Junping Du via wheat9)
+ HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
+ (Inigo Goiri via cdouglas)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8652cce5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index fdea387..5d02fcf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@@ -2722,8 +2724,20 @@ public abstract class FileSystem extends Configured implements Closeable {
synchronized (FileSystem.class) {
if (!FILE_SYSTEMS_LOADED) {
ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
- for (FileSystem fs : serviceLoader) {
- SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ Iterator<FileSystem> it = serviceLoader.iterator();
+ while (it.hasNext()) {
+ FileSystem fs = null;
+ try {
+ fs = it.next();
+ try {
+ SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ } catch (Exception e) {
+ LOG.warn("Cannot load: " + fs + " from " +
+ ClassUtil.findContainingJar(fs.getClass()), e);
+ }
+ } catch (ServiceConfigurationError ee) {
+ LOG.warn("Cannot load filesystem", ee);
+ }
}
FILE_SYSTEMS_LOADED = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8652cce5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.net.URL;
+import java.util.ServiceConfigurationError;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
assertFalse(false);
}
}
+
+ @Test
+ public void testMissingLibraries() {
+ boolean catched = false;
+ try {
+ Configuration conf = new Configuration();
+ FileSystem.getFileSystemClass("s3a", conf);
+ } catch (Exception e) {
+ catched = true;
+ } catch (ServiceConfigurationError e) {
+ // S3A shouldn't find AWS SDK and fail
+ catched = true;
+ }
+ assertTrue(catched);
+ }
}
[3/3] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure
init for unused FileSystems. Contributed by Inigo Goiri
Posted by cd...@apache.org.
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri
(cherry picked from commit 8652cce5b21825f6f835e4ea31de82eb59fb06c1)
(cherry picked from commit d984ba4fd73df64b7dca1ec10b35956e4ae9bd57)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4150d08a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4150d08a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4150d08a
Branch: refs/heads/branch-2.8
Commit: 4150d08a4b44d2f61cd183fbeb4523721a5c2a77
Parents: d6f1d3b
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Dec 18 18:31:36 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++--
.../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++--
.../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++
3 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e22da88..247124f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -765,8 +765,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-12656. MiniKdc throws "address in use" BindException.
(Wei-Chiu Chuang via Arpit Agarwal)
- OPTIMIZATIONS
-
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
over getMessage() in logging/span events. (Varun Saxena via stevel)
@@ -882,6 +880,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
server. (Junping Du via wheat9)
+ HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
+ (Inigo Goiri via cdouglas)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 8ed96bf..2102862 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable {
synchronized (FileSystem.class) {
if (!FILE_SYSTEMS_LOADED) {
ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
- for (FileSystem fs : serviceLoader) {
- SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ Iterator<FileSystem> it = serviceLoader.iterator();
+ while (it.hasNext()) {
+ FileSystem fs = null;
+ try {
+ fs = it.next();
+ try {
+ SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ } catch (Exception e) {
+ LOG.warn("Cannot load: " + fs + " from " +
+ ClassUtil.findContainingJar(fs.getClass()), e);
+ }
+ } catch (ServiceConfigurationError ee) {
+ LOG.warn("Cannot load filesystem", ee);
+ }
}
FILE_SYSTEMS_LOADED = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4150d08a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.net.URL;
+import java.util.ServiceConfigurationError;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
assertFalse(false);
}
}
+
+ @Test
+ public void testMissingLibraries() {
+ boolean catched = false;
+ try {
+ Configuration conf = new Configuration();
+ FileSystem.getFileSystemClass("s3a", conf);
+ } catch (Exception e) {
+ catched = true;
+ } catch (ServiceConfigurationError e) {
+ // S3A shouldn't find AWS SDK and fail
+ catched = true;
+ }
+ assertTrue(catched);
+ }
}
[2/3] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure
init for unused FileSystems. Contributed by Inigo Goiri
Posted by cd...@apache.org.
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri
(cherry picked from commit 8652cce5b21825f6f835e4ea31de82eb59fb06c1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d984ba4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d984ba4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d984ba4f
Branch: refs/heads/branch-2
Commit: d984ba4fd73df64b7dca1ec10b35956e4ae9bd57
Parents: dd91cf9
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Fri Dec 18 18:31:20 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++--
.../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++--
.../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++
3 files changed, 35 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d984ba4f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e4903fc..e578d8f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -782,8 +782,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-12656. MiniKdc throws "address in use" BindException.
(Wei-Chiu Chuang via Arpit Agarwal)
- OPTIMIZATIONS
-
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
over getMessage() in logging/span events. (Varun Saxena via stevel)
@@ -899,6 +897,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10729. Add tests for PB RPC in case version mismatch of client and
server. (Junping Du via wheat9)
+ HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems.
+ (Inigo Goiri via cdouglas)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d984ba4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 8ed96bf..2102862 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -36,6 +36,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
@@ -62,6 +63,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@@ -2708,8 +2710,20 @@ public abstract class FileSystem extends Configured implements Closeable {
synchronized (FileSystem.class) {
if (!FILE_SYSTEMS_LOADED) {
ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
- for (FileSystem fs : serviceLoader) {
- SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ Iterator<FileSystem> it = serviceLoader.iterator();
+ while (it.hasNext()) {
+ FileSystem fs = null;
+ try {
+ fs = it.next();
+ try {
+ SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ } catch (Exception e) {
+ LOG.warn("Cannot load: " + fs + " from " +
+ ClassUtil.findContainingJar(fs.getClass()), e);
+ }
+ } catch (ServiceConfigurationError ee) {
+ LOG.warn("Cannot load filesystem", ee);
+ }
}
FILE_SYSTEMS_LOADED = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d984ba4f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.net.URL;
+import java.util.ServiceConfigurationError;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
assertFalse(false);
}
}
+
+ @Test
+ public void testMissingLibraries() {
+ boolean catched = false;
+ try {
+ Configuration conf = new Configuration();
+ FileSystem.getFileSystemClass("s3a", conf);
+ } catch (Exception e) {
+ catched = true;
+ } catch (ServiceConfigurationError e) {
+ // S3A shouldn't find AWS SDK and fail
+ catched = true;
+ }
+ assertTrue(catched);
+ }
}