You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2016/07/11 22:32:03 UTC
[1/2] hadoop git commit: HADOOP-12636. Prevent ServiceLoader failure
init for unused FileSystems. Contributed by Inigo Goiri. Cherry picked from
commit #8652cce5b2
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7.3 01aa7d8f2 -> 490bcda71
HADOOP-12636. Prevent ServiceLoader failure init for unused FileSystems. Contributed by Inigo Goiri.
Cherry picked from commit #8652cce5b2
(cherry picked from commit 7595d89612f4916ab29a6ac4e07374771fcbc94f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcd058ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcd058ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcd058ed
Branch: refs/heads/branch-2.7.3
Commit: fcd058edb60c7a02ad07d0fb8ccf85befe212308
Parents: 01aa7d8
Author: Chris Douglas <cd...@apache.org>
Authored: Fri Dec 18 18:21:52 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Jul 11 15:28:01 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 18 ++++++++++++++++--
.../hadoop/fs/TestFileSystemInitialization.java | 16 ++++++++++++++++
2 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd058ed/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index fac3c40..2e684f5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -35,6 +35,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.Stack;
@@ -61,6 +62,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ClassUtil;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
@@ -2622,8 +2624,20 @@ public abstract class FileSystem extends Configured implements Closeable {
synchronized (FileSystem.class) {
if (!FILE_SYSTEMS_LOADED) {
ServiceLoader<FileSystem> serviceLoader = ServiceLoader.load(FileSystem.class);
- for (FileSystem fs : serviceLoader) {
- SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ Iterator<FileSystem> it = serviceLoader.iterator();
+ while (it.hasNext()) {
+ FileSystem fs = null;
+ try {
+ fs = it.next();
+ try {
+ SERVICE_FILE_SYSTEMS.put(fs.getScheme(), fs.getClass());
+ } catch (Exception e) {
+ LOG.warn("Cannot load: " + fs + " from " +
+ ClassUtil.findContainingJar(fs.getClass()), e);
+ }
+ } catch (ServiceConfigurationError ee) {
+ LOG.warn("Cannot load filesystem", ee);
+ }
}
FILE_SYSTEMS_LOADED = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcd058ed/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
index d3fceec..18e8b01 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemInitialization.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import java.io.IOException;
import java.net.URL;
+import java.util.ServiceConfigurationError;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -43,4 +44,19 @@ public class TestFileSystemInitialization {
assertFalse(false);
}
}
+
+ @Test
+ public void testMissingLibraries() {
+ boolean catched = false;
+ try {
+ Configuration conf = new Configuration();
+ FileSystem.getFileSystemClass("s3a", conf);
+ } catch (Exception e) {
+ catched = true;
+ } catch (ServiceConfigurationError e) {
+ // S3A shouldn't find AWS SDK and fail
+ catched = true;
+ }
+ assertTrue(catched);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDFS-10488. Update WebHDFS documentation
regarding CREATE and MKDIR default permissions. Contributed by Wellington
Chevreuil.
Posted by vi...@apache.org.
HDFS-10488. Update WebHDFS documentation regarding CREATE and MKDIR default permissions. Contributed by Wellington Chevreuil.
(cherry picked from commit 728623242f47c6bffb46cfcc8f3425b58c1fde3a)
(cherry picked from commit 5ebebc652454fd5541d1a8231ba7ac2656e36cba)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/490bcda7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/490bcda7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/490bcda7
Branch: refs/heads/branch-2.7.3
Commit: 490bcda71fc607dbc9d2785695e43f251571bb24
Parents: fcd058e
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Jul 7 10:36:18 2016 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Jul 11 15:28:37 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/490bcda7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index efab9eb..62d3237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -265,6 +265,8 @@ File and Directory Operations
Location: webhdfs://<HOST>:<PORT>/<PATH>
Content-Length: 0
+If no permissions are specified, the newly created file will be assigned with default 755 permission. This also applies for new directories. No umask mode will be applied from server side (so "fs.permissions.umask-mode" value configuration set on Namenode side will have no effect).
+
**Note** that the reason of having two-step create/append is for preventing clients to send out data before the redirect. This issue is addressed by the "`Expect: 100-continue`" header in HTTP/1.1; see [RFC 2616, Section 8.2.3](http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html#sec8.2.3). Unfortunately, there are software library bugs (e.g. Jetty 6 HTTP server and Java 6 HTTP client), which do not correctly implement "`Expect: 100-continue`". The two-step create/append is a temporary workaround for the software library bugs.
See also: [`overwrite`](#Overwrite), [`blocksize`](#Block_Size), [`replication`](#Replication), [`permission`](#Permission), [`buffersize`](#Buffer_Size), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create
@@ -344,6 +346,8 @@ See also: [`offset`](#Offset), [`length`](#Length), [`buffersize`](#Buffer_Size)
{"boolean": true}
+If no permissions are specified, the newly created directory will be assigned with default 755 permission. This also applies to new files created. No umask mode will be applied from server side (so "fs.permissions.umask-mode" value configuration set on Namenode side will have no effect).
+
See also: [`permission`](#Permission), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs
### Create a Symbolic Link
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org