You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2013/10/02 02:33:20 UTC
svn commit: r1528285 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common:
./ src/main/java/org/apache/hadoop/fs/ src/main/resources/
Author: cmccabe
Date: Wed Oct 2 00:33:20 2013
New Revision: 1528285
URL: http://svn.apache.org/r1528285
Log:
HADOOP-9758. Provide configuration option for FS/FC symlink resolution. (Andrew Wang via Colin Patrick McCabe)
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 2 00:33:20 2013
@@ -29,6 +29,9 @@ Release 2.1.2 - UNRELEASED
HADOOP-9976. Different versions of avro and avro-maven-plugin (Karthik
Kambatla via Sandy Ryza)
+ HADOOP-9758. Provide configuration option for FileSystem/FileContext
+ symlink resolution (Andrew Wang via Colin Patrick McCabe)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java Wed Oct 2 00:33:20 2013
@@ -54,7 +54,11 @@ public class CommonConfigurationKeysPubl
public static final String FS_DF_INTERVAL_KEY = "fs.df.interval";
/** Default value for FS_DF_INTERVAL_KEY */
public static final long FS_DF_INTERVAL_DEFAULT = 60000;
-
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY =
+ "fs.client.resolve.remote.symlinks";
+ /** Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY */
+ public static final boolean FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT = true;
//Defaults are not specified for following keys
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java Wed Oct 2 00:33:20 2013
@@ -90,6 +90,11 @@ public abstract class FSLinkResolver<T>
in = next(fs, p);
isLink = false;
} catch (UnresolvedLinkException e) {
+ if (!fc.resolveSymlinks) {
+ throw new IOException("Path " + path + " contains a symlink"
+ + " and symlink resolution is disabled ("
+ + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e);
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java Wed Oct 2 00:33:20 2013
@@ -225,6 +225,7 @@ public final class FileContext {
private FsPermission umask;
private final Configuration conf;
private final UserGroupInformation ugi;
+ final boolean resolveSymlinks;
private FileContext(final AbstractFileSystem defFs,
final FsPermission theUmask, final Configuration aConf) {
@@ -250,9 +251,12 @@ public final class FileContext {
if (workingDir == null) {
workingDir = defaultFS.getHomeDirectory();
}
+ resolveSymlinks = conf.getBoolean(
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
util = new Util(); // for the inner class
}
-
+
/*
* Remove relative part - return "absolute":
* If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Wed Oct 2 00:33:20 2013
@@ -120,6 +120,7 @@ public abstract class FileSystem extends
*/
private Set<Path> deleteOnExit = new TreeSet<Path>();
+ boolean resolveSymlinks;
/**
* This method adds a file system for testing so that we can find it later. It
* is only for testing.
@@ -196,6 +197,9 @@ public abstract class FileSystem extends
*/
public void initialize(URI name, Configuration conf) throws IOException {
statistics = getStatistics(name.getScheme(), getClass());
+ resolveSymlinks = conf.getBoolean(
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
}
/**
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java Wed Oct 2 00:33:20 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -80,6 +81,12 @@ public abstract class FileSystemLinkReso
in = doCall(p);
isLink = false;
} catch (UnresolvedLinkException e) {
+ if (!filesys.resolveSymlinks) {
+ throw new IOException("Path " + path + " contains a symlink"
+ + " and symlink resolution is disabled ("
+ + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY
+ + ").", e);
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1528285&r1=1528284&r2=1528285&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Wed Oct 2 00:33:20 2013
@@ -1203,6 +1203,17 @@
</property>
<property>
+ <name>fs.client.resolve.remote.symlinks</name>
+ <value>true</value>
+ <description>
+ Whether to resolve symlinks when accessing a remote Hadoop filesystem.
+ Setting this to false causes an exception to be thrown upon encountering
+ a symlink. This setting does not apply to local filesystems, which
+ automatically resolve local symlinks.
+ </description>
+</property>
+
+<property>
<name>nfs3.server.port</name>
<value>2049</value>
<description>