You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/27 23:26:12 UTC
[09/50] [abbrv] hadoop git commit: HADOOP-11852. Disable symlinks in
trunk.
HADOOP-11852. Disable symlinks in trunk.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aa6fec33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aa6fec33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aa6fec33
Branch: refs/heads/YARN-2928
Commit: aa6fec334889c32afb9748d075c656ce33ab8b49
Parents: f62eaf6
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Apr 23 11:47:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 27 14:18:47 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../java/org/apache/hadoop/fs/FSLinkResolver.java | 4 ++++
.../main/java/org/apache/hadoop/fs/FileContext.java | 4 ++++
.../main/java/org/apache/hadoop/fs/FileSystem.java | 15 +++++++++++++++
.../org/apache/hadoop/fs/FileSystemLinkResolver.java | 4 ++++
.../org/apache/hadoop/fs/RawLocalFileSystem.java | 4 ++++
.../java/org/apache/hadoop/fs/SymlinkBaseTest.java | 4 ++++
.../apache/hadoop/fs/TestFileContextResolveAfs.java | 4 +++-
.../src/test/java/org/apache/hadoop/fs/TestStat.java | 4 +++-
.../apache/hadoop/hdfs/DistributedFileSystem.java | 4 ++++
.../hadoop/hdfs/server/namenode/FSEditLogLoader.java | 4 ++++
.../hadoop/hdfs/server/namenode/FSImageFormat.java | 5 +++++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++++
.../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +++
.../hadoop/hdfs/server/namenode/TestINodeFile.java | 4 ++++
15 files changed, 67 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index db1425f..f232e04 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -201,6 +201,8 @@ Trunk (Unreleased)
HADOOP-11850. Typos in hadoop-common java docs. (Surendra Singh Lilhore
via jghoman)
+ HADOOP-11852. Disable symlinks in trunk.
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
index 831d4ca..ffe4b34 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
@@ -95,6 +95,10 @@ public abstract class FSLinkResolver<T> {
+ " and symlink resolution is disabled ("
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e);
}
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new IOException("Symlink resolution is disabled in"
+ + " this version of Hadoop.");
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0b5863b..ea3f896 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -1431,11 +1431,15 @@ public class FileContext {
* <code>target</code> or <code>link</code> is not supported
* @throws IOException If an I/O error occurred
*/
+ @SuppressWarnings("deprecation")
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
ParentNotDirectoryException, UnsupportedFileSystemException,
IOException {
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new UnsupportedOperationException("Symlinks not supported");
+ }
final Path nonRelLink = fixRelativePart(link);
new FSLinkResolver<Void>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 79b7403..b508727 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -3296,4 +3296,19 @@ public abstract class FileSystem extends Configured implements Closeable {
": " + pair.getValue());
}
}
+
+ // Symlinks are temporarily disabled - see HADOOP-10020 and HADOOP-10052
+ private static boolean symlinksEnabled = false;
+
+ private static Configuration conf = null;
+
+ @VisibleForTesting
+ public static boolean areSymlinksEnabled() {
+ return symlinksEnabled;
+ }
+
+ @VisibleForTesting
+ public static void enableSymlinks() {
+ symlinksEnabled = true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
index c01d41f..7eec0eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
@@ -87,6 +87,10 @@ public abstract class FileSystemLinkResolver<T> {
+ CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY
+ ").", e);
}
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new IOException("Symlink resolution is disabled in" +
+ " this version of Hadoop.");
+ }
if (count++ > FsConstants.MAX_PATH_LINKS) {
throw new IOException("Possible cyclic loop while " +
"following symbolic link " + path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
index 52623b8..56dd7ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
@@ -775,9 +775,13 @@ public class RawLocalFileSystem extends FileSystem {
return true;
}
+ @SuppressWarnings("deprecation")
@Override
public void createSymlink(Path target, Path link, boolean createParent)
throws IOException {
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new UnsupportedOperationException("Symlinks not supported");
+ }
final String targetScheme = target.toUri().getScheme();
if (targetScheme != null && !"file".equals(targetScheme)) {
throw new IOException("Unable to create symlink to non-local file "+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index 9fe2cd7..4d6485d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -36,6 +36,10 @@ import org.junit.After;
* Base test for symbolic links
*/
public abstract class SymlinkBaseTest {
+ // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052
+ static {
+ FileSystem.enableSymlinks();
+ }
static final long seed = 0xDEADBEEFL;
static final int blockSize = 8192;
static final int fileSize = 16384;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java
index ca9de83..1fe82e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java
@@ -31,7 +31,9 @@ import org.junit.Test;
* Tests resolution of AbstractFileSystems for a given path with symlinks.
*/
public class TestFileContextResolveAfs {
-
+ static {
+ FileSystem.enableSymlinks();
+ }
private static String TEST_ROOT_DIR_LOCAL
= System.getProperty("test.build.data","/tmp");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
index 6265012..0e518e2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestStat.java
@@ -32,7 +32,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class TestStat extends FileSystemTestHelper {
-
+ static {
+ FileSystem.enableSymlinks();
+ }
private static Stat stat;
@BeforeClass
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 8d6a8fe..4ca6d57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1286,12 +1286,16 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ @SuppressWarnings("deprecation")
@Override
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws AccessControlException,
FileAlreadyExistsException, FileNotFoundException,
ParentNotDirectoryException, UnsupportedFileSystemException,
IOException {
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new UnsupportedOperationException("Symlinks not supported");
+ }
statistics.incrementWriteOps(1);
final Path absF = fixRelativePart(link);
new FileSystemLinkResolver<Void>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 293bcba..43171de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.hdfs.protocol.HdfsConstantsClient;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -630,6 +631,9 @@ public class FSEditLogLoader {
break;
}
case OP_SYMLINK: {
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS");
+ }
SymlinkOp symlinkOp = (SymlinkOp)op;
inodeId = getAndUpdateLastInodeId(symlinkOp.inodeId, logVersion,
lastInodeId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index cce991f..906213e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -42,6 +42,7 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
@@ -723,6 +724,7 @@ public class FSImageFormat {
* @param counter Counter to increment for namenode startup progress
* @return an inode
*/
+ @SuppressWarnings("deprecation")
INode loadINode(final byte[] localName, boolean isSnapshotINode,
DataInput in, Counter counter) throws IOException {
final int imgVersion = getLayoutVersion();
@@ -836,6 +838,9 @@ public class FSImageFormat {
return dir;
} else if (numBlocks == -2) {
//symlink
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new IOException("Symlinks not supported - please remove symlink before upgrading to this version of HDFS");
+ }
final String symlink = Text.readString(in);
final PermissionStatus permissions = PermissionStatus.read(in);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e400644..3599fad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -147,6 +147,7 @@ import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Options;
@@ -2131,6 +2132,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
void createSymlink(String target, String link,
PermissionStatus dirPerms, boolean createParent, boolean logRetryCache)
throws IOException {
+ if (!FileSystem.areSymlinksEnabled()) {
+ throw new UnsupportedOperationException("Symlinks not supported");
+ }
waitForLoadingFSImage();
HdfsFileStatus auditStat = null;
checkOperation(OperationCategory.WRITE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index d3eaa6e..8658e47 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -769,6 +769,9 @@ public class MiniDFSCluster {
try {
ExitUtil.disableSystemExit();
+ // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052
+ FileSystem.enableSymlinks();
+
synchronized (MiniDFSCluster.class) {
instanceId = instanceCount++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aa6fec33/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index 8e1f1ef..4e6c59a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -71,6 +71,10 @@ import org.mockito.Mockito;
import com.google.common.collect.ImmutableList;
public class TestINodeFile {
+ // Re-enable symlinks for tests, see HADOOP-10020 and HADOOP-10052
+ static {
+ FileSystem.enableSymlinks();
+ }
public static final Log LOG = LogFactory.getLog(TestINodeFile.class);
static final short BLOCKBITS = 48;