You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sr...@apache.org on 2010/02/16 22:43:32 UTC
svn commit: r910706 [1/2] - in /hadoop/common/trunk: ./
src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/fs/local/
src/java/org/apache/hadoop/util/ src/test/core/org/apache/hadoop/fs/
Author: sradia
Date: Tue Feb 16 21:43:30 2010
New Revision: 910706
URL: http://svn.apache.org/viewvc?rev=910706&view=rev
Log:
HADOOP-6421 Adds Symbolic links to FileContext, AbstractFileSystem.
It also adds a limited implementation for the local file system
(RawLocalFs) that allows local symlinks. (Eli Collins via Sanjay Radia)
Added:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/UnresolvedLinkException.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextSymlinkBaseTest.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestLocalFSFileContextSymlink.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/DelegateToFileSystem.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileStatus.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/Path.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/local/RawLocalFs.java
hadoop/common/trunk/src/java/org/apache/hadoop/util/Shell.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestPath.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Tue Feb 16 21:43:30 2010
@@ -54,6 +54,10 @@
HADOOP-6510. Adds a way for superusers to impersonate other users
in a secure environment. (Jitendra Nath Pandey via ddas)
+ HADOOP-6421 Adds Symbolic links to FileContext, AbstractFileSystem.
+ It also adds a limited implementation for the local file system
+ (RawLocalFs) that allows local symlinks. (Eli Collins via Sanjay Radia)
+
IMPROVEMENTS
HADOOP-6283. Improve the exception messages thrown by
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/AbstractFileSystem.java Tue Feb 16 21:43:30 2010
@@ -77,7 +77,7 @@
}
/**
- * Prohibits names which contain a ".", "..". ":" or "/"
+ * Prohibits names which contain a ".", "..", ":" or "/"
*/
private static boolean isValidName(String src) {
// Check for ".." "." ":" "/"
@@ -352,7 +352,7 @@
* @return server default configuration values
* @throws IOException
*/
- protected abstract FsServerDefaults getServerDefaults() throws IOException;
+ protected abstract FsServerDefaults getServerDefaults() throws IOException;
/**
* The specification of this method matches that of
@@ -362,7 +362,7 @@
*/
protected final FSDataOutputStream create(final Path f,
final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(f);
int bufferSize = -1;
short replication = -1;
@@ -457,7 +457,8 @@
protected abstract FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent) throws IOException;
+ int bytesPerChecksum, boolean createParent)
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -467,7 +468,7 @@
*/
protected abstract void mkdir(final Path dir,
final FsPermission permission, final boolean createParent)
- throws IOException;
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -475,14 +476,15 @@
* this filesystem.
*/
protected abstract boolean delete(final Path f, final boolean recursive)
- throws IOException;
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
* {@link FileContext#open(Path)} except that Path f must be for this
* filesystem.
*/
- protected FSDataInputStream open(final Path f) throws IOException {
+ protected FSDataInputStream open(final Path f)
+ throws IOException, UnresolvedLinkException {
return open(f, getServerDefaults().getFileBufferSize());
}
@@ -490,9 +492,10 @@
* The specification of this method matches that of
* {@link FileContext#open(Path, int)} except that Path f must be for this
* filesystem.
+ * @throws UnresolvedLinkException
*/
protected abstract FSDataInputStream open(final Path f, int bufferSize)
- throws IOException;
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -500,7 +503,7 @@
* for this filesystem.
*/
protected abstract boolean setReplication(final Path f,
- final short replication) throws IOException;
+ final short replication) throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -508,7 +511,8 @@
* f must be for this filesystem.
*/
protected final void rename(final Path src, final Path dst,
- final Options.Rename... options) throws IOException {
+ final Options.Rename... options)
+ throws IOException, UnresolvedLinkException {
boolean overwrite = false;
if (null != options) {
for (Rename option : options) {
@@ -530,7 +534,7 @@
* {@link #renameInternal(Path, Path, boolean)}
*/
protected abstract void renameInternal(final Path src, final Path dst)
- throws IOException;
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -538,16 +542,16 @@
* f must be for this filesystem.
*/
protected void renameInternal(final Path src, final Path dst,
- boolean overwrite) throws IOException {
+ boolean overwrite) throws IOException, UnresolvedLinkException {
// Default implementation deals with overwrite in a non-atomic way
- final FileStatus srcStatus = getFileStatus(src);
+ final FileStatus srcStatus = getFileLinkStatus(src);
if (srcStatus == null) {
throw new FileNotFoundException("rename source " + src + " not found.");
}
FileStatus dstStatus;
try {
- dstStatus = getFileStatus(dst);
+ dstStatus = getFileLinkStatus(dst);
} catch (IOException e) {
dstStatus = null;
}
@@ -571,12 +575,12 @@
delete(dst, false);
} else {
final Path parent = dst.getParent();
- final FileStatus parentStatus = getFileStatus(parent);
+ final FileStatus parentStatus = getFileLinkStatus(parent);
if (parentStatus == null) {
throw new FileNotFoundException("rename destination parent " + parent
+ " not found.");
}
- if (!parentStatus.isDir()) {
+ if (!parentStatus.isDir() && !parentStatus.isSymlink()) {
throw new ParentNotDirectoryException("rename destination parent "
+ parent + " is a file.");
}
@@ -585,12 +589,41 @@
}
/**
+ * Returns true if the file system supports symlinks, false otherwise.
+ */
+ protected boolean supportsSymlinks() {
+ return false;
+ }
+
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#createSymlink(Path, Path, boolean)};
+ */
+ protected void createSymlink(final Path target, final Path link,
+ final boolean createParent) throws IOException, UnresolvedLinkException {
+ throw new IOException("File system does not support symlinks");
+ }
+
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#getLinkTarget(Path)};
+ */
+ protected Path getLinkTarget(final Path f) throws IOException {
+ /* We should never get here. Any file system that threw an
+ * UnresolvedLinkException, causing this function to be called,
+ * needs to override this method.
+ */
+ throw new AssertionError();
+ }
+
+ /**
* The specification of this method matches that of
* {@link FileContext#setPermission(Path, FsPermission)} except that Path f
* must be for this filesystem.
*/
protected abstract void setPermission(final Path f,
- final FsPermission permission) throws IOException;
+ final FsPermission permission)
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -598,7 +631,7 @@
* be for this filesystem.
*/
protected abstract void setOwner(final Path f, final String username,
- final String groupname) throws IOException;
+ final String groupname) throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -606,7 +639,7 @@
* for this filesystem.
*/
protected abstract void setTimes(final Path f, final long mtime,
- final long atime) throws IOException;
+ final long atime) throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
@@ -614,14 +647,29 @@
* this filesystem.
*/
protected abstract FileChecksum getFileChecksum(final Path f)
- throws IOException;
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
- * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
- * must be for this filesystem.
+ * {@link FileContext#getFileStatus(Path)}
+ * except that an UnresolvedLinkException may be thrown if a symlink is
+ * encountered in the path.
*/
- protected abstract FileStatus getFileStatus(final Path f) throws IOException;
+ protected abstract FileStatus getFileStatus(final Path f)
+ throws IOException, UnresolvedLinkException;
+
+ /**
+ * The specification of this method matches that of
+ * {@link FileContext#getFileLinkStatus(Path)}
+ * except that an UnresolvedLinkException may be thrown if a symlink is
+ * encountered in the path leading up to the final path component.
+ * If the file system does not support symlinks then the behavior is
+ * equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
+ */
+ protected FileStatus getFileLinkStatus(final Path f)
+ throws IOException, UnresolvedLinkException {
+ return getFileStatus(f);
+ }
/**
* The specification of this method matches that of
@@ -629,22 +677,23 @@
* Path f must be for this filesystem.
*/
protected abstract BlockLocation[] getFileBlockLocations(final Path f,
- final long start, final long len) throws IOException;
+ final long start, final long len)
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
* {@link FileContext#getFsStatus(Path)} except that Path f must be for this
* filesystem.
*/
- protected FsStatus getFsStatus(final Path f) throws IOException {
+ protected FsStatus getFsStatus(final Path f)
+ throws IOException, UnresolvedLinkException {
// default impl gets FsStatus of root
return getFsStatus();
}
/**
* The specification of this method matches that of
- * {@link FileContext#getFsStatus(Path)} except that Path f must be for this
- * filesystem.
+ * {@link FileContext#getFsStatus(Path)}.
*/
protected abstract FsStatus getFsStatus() throws IOException;
@@ -653,7 +702,8 @@
* {@link FileContext#listStatus(Path)} except that Path f must be for this
* filesystem.
*/
- protected abstract FileStatus[] listStatus(final Path f) throws IOException;
+ protected abstract FileStatus[] listStatus(final Path f)
+ throws IOException, UnresolvedLinkException;
/**
* The specification of this method matches that of
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java Tue Feb 16 21:43:30 2010
@@ -115,12 +115,12 @@
private long fileLen = -1L;
public ChecksumFSInputChecker(ChecksumFs fs, Path file)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
this(fs, file, fs.getServerDefaults().getFileBufferSize());
}
public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
super(file, fs.getFileStatus(file).getReplication());
this.datas = fs.getRawFs().open(file, bufferSize);
this.fs = fs;
@@ -160,7 +160,7 @@
}
public int read(long position, byte[] b, int off, int len)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
// parameter check
if ((off | len | (off + len) | (b.length - (off + len))) < 0) {
throw new IndexOutOfBoundsException();
@@ -236,7 +236,7 @@
}
/* Return the file length */
- private long getFileLength() throws IOException {
+ private long getFileLength() throws IOException, UnresolvedLinkException {
if (fileLen==-1L) {
fileLen = fs.getFileStatus(file).getLen();
}
@@ -257,7 +257,7 @@
* @exception IOException if an I/O error occurs.
* ChecksumException if the chunk to skip to is corrupted
*/
- public synchronized long skip(long n) throws IOException {
+ public synchronized long skip(long n) throws IOException {
final long curPos = getPos();
final long fileLength = getFileLength();
if (n+curPos > fileLength) {
@@ -278,7 +278,7 @@
* ChecksumException if the chunk to seek to is corrupted
*/
- public synchronized void seek(long pos) throws IOException {
+ public synchronized void seek(long pos) throws IOException {
if (pos>getFileLength()) {
throw new IOException("Cannot seek after EOF");
}
@@ -293,7 +293,8 @@
* @param bufferSize the size of the buffer to be used.
*/
@Override
- public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ public FSDataInputStream open(Path f, int bufferSize)
+ throws IOException, UnresolvedLinkException {
return new FSDataInputStream(
new ChecksumFSInputChecker(this, f, bufferSize));
}
@@ -371,7 +372,8 @@
/** Check if exists.
* @param f source file
*/
- private boolean exists(Path f) throws IOException {
+ private boolean exists(Path f)
+ throws IOException, UnresolvedLinkException {
try {
return getMyFs().getFileStatus(f) != null;
} catch (FileNotFoundException e) {
@@ -383,7 +385,8 @@
* Note: Avoid using this method. Instead reuse the FileStatus
* returned by getFileStatus() or listStatus() methods.
*/
- private boolean isDirectory(Path f) throws IOException {
+ private boolean isDirectory(Path f)
+ throws IOException, UnresolvedLinkException {
try {
return getMyFs().getFileStatus(f).isDir();
} catch (FileNotFoundException e) {
@@ -401,7 +404,7 @@
*/
@Override
public boolean setReplication(Path src, short replication)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
boolean value = getMyFs().setReplication(src, replication);
if (!value) {
return false;
@@ -417,7 +420,8 @@
* Rename files/dirs.
*/
@Override
- public void renameInternal(Path src, Path dst) throws IOException {
+ public void renameInternal(Path src, Path dst)
+ throws IOException, UnresolvedLinkException {
if (isDirectory(src)) {
getMyFs().rename(src, dst);
} else {
@@ -438,7 +442,8 @@
* Implement the delete(Path, boolean) in checksum
* file system.
*/
- public boolean delete(Path f, boolean recursive) throws IOException{
+ public boolean delete(Path f, boolean recursive)
+ throws IOException, UnresolvedLinkException {
FileStatus fstatus = null;
try {
fstatus = getMyFs().getFileStatus(f);
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/DelegateToFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/DelegateToFileSystem.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/DelegateToFileSystem.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/DelegateToFileSystem.java Tue Feb 16 21:43:30 2010
@@ -26,6 +26,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Progressable;
@@ -107,6 +108,11 @@
}
@Override
+ protected FileStatus getFileLinkStatus(final Path f) throws IOException {
+ return getFileStatus(f);
+ }
+
+ @Override
protected FsStatus getFsStatus() throws IOException {
return fsImpl.getStatus();
}
@@ -148,7 +154,6 @@
checkPath(src);
checkPath(dst);
fsImpl.rename(src, dst, Options.Rename.NONE);
-
}
@Override
@@ -156,7 +161,6 @@
throws IOException {
checkPath(f);
fsImpl.setOwner(f, username, groupname);
-
}
@Override
@@ -177,11 +181,30 @@
protected void setTimes(Path f, long mtime, long atime) throws IOException {
checkPath(f);
fsImpl.setTimes(f, mtime, atime);
-
}
@Override
protected void setVerifyChecksum(boolean verifyChecksum) throws IOException {
fsImpl.setVerifyChecksum(verifyChecksum);
}
-}
+
+ @Override
+ protected boolean supportsSymlinks() {
+ return false;
+ }
+
+ @Override
+ protected void createSymlink(Path target, Path link, boolean createParent)
+ throws IOException {
+ throw new IOException("File system does not support symlinks");
+ }
+
+ @Override
+ protected Path getLinkTarget(final Path f) throws IOException {
+ /* We should never get here. Any file system that threw an
+ * UnresolvedLinkException, causing this function to be called,
+ * should override getLinkTarget.
+ */
+ throw new AssertionError();
+ }
+}
\ No newline at end of file
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Tue Feb 16 21:43:30 2010
@@ -40,7 +40,6 @@
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.CreateOpts;
-import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
@@ -142,7 +141,8 @@
* Generally you should not need use a config unless you are doing
* <ul>
* <li> configX = someConfigSomeOnePassedToYou.
- * <li> myFContext = getFileContext(configX); //configX not changed but passeddown
+ * <li> myFContext = getFileContext(configX); // configX is not changed,
+ * // is passed down
* <li> myFContext.create(path, ...);
* <li>...
* </ul>
@@ -213,15 +213,15 @@
*
* Applications that use FileContext should use #makeQualified() since
* they really want a fully qualified URI.
- * Hence this method os not called makeAbsolute() and
+ * Hence this method is not called makeAbsolute() and
* has been deliberately declared private.
*/
- private Path fixRelativePart(Path f) {
- if (f.isUriPathAbsolute()) {
- return f;
+ private Path fixRelativePart(Path p) {
+ if (p.isUriPathAbsolute()) {
+ return p;
} else {
- return new Path(workingDir, f);
+ return new Path(workingDir, p);
}
}
@@ -429,12 +429,14 @@
*/
public void setWorkingDirectory(final Path newWDir) throws IOException {
checkNotSchemeWithRelative(newWDir);
- // wd is stored as fully qualified path.
-
- final Path newWorkingDir = new Path(workingDir, newWDir);
+ /* wd is stored as a fully qualified path. We check if the given
+ * path is not relative first since resolve requires and returns
+ * an absolute path.
+ */
+ final Path newWorkingDir = resolve(new Path(workingDir, newWDir));
FileStatus status = getFileStatus(newWorkingDir);
if (!status.isDir()) {
- throw new FileNotFoundException(" Cannot setWD to a file");
+ throw new FileNotFoundException("Cannot setWD to a file");
}
workingDir = newWorkingDir;
}
@@ -510,7 +512,6 @@
Options.CreateOpts... opts)
throws IOException {
Path absF = fixRelativePart(f);
- AbstractFileSystem fsOfAbsF = getFSofPath(absF);
// If one of the options is a permission, extract it & apply umask
// If not, add a default Perms and apply umask;
@@ -522,9 +523,14 @@
FsPermission.getDefault();
permission = permission.applyUMask(umask);
- CreateOpts[] updatedOpts =
+ final CreateOpts[] updatedOpts =
CreateOpts.setOpt(CreateOpts.perms(permission), opts);
- return fsOfAbsF.create(absF, createFlag, updatedOpts);
+ return new FSLinkResolver<FSDataOutputStream>() {
+ public FSDataOutputStream next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.create(p, createFlag, updatedOpts);
+ }
+ }.resolve(this, absF);
}
/**
@@ -541,10 +547,16 @@
public void mkdir(final Path dir, final FsPermission permission,
final boolean createParent)
throws IOException {
- Path absDir = fixRelativePart(dir);
- FsPermission absFerms = (permission == null ?
+ final Path absDir = fixRelativePart(dir);
+ final FsPermission absFerms = (permission == null ?
FsPermission.getDefault() : permission).applyUMask(umask);
- getFSofPath(absDir).mkdir(absDir, absFerms, createParent);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.mkdir(p, absFerms, createParent);
+ return null;
+ }
+ }.resolve(this, absDir);
}
/**
@@ -559,7 +571,12 @@
public boolean delete(final Path f, final boolean recursive)
throws IOException {
Path absF = fixRelativePart(f);
- return getFSofPath(absF).delete(absF, recursive);
+ return new FSLinkResolver<Boolean>() {
+ public Boolean next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return Boolean.valueOf(fs.delete(p, recursive));
+ }
+ }.resolve(this, absF);
}
/**
@@ -569,7 +586,12 @@
*/
public FSDataInputStream open(final Path f) throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).open(absF);
+ return new FSLinkResolver<FSDataInputStream>() {
+ public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.open(p);
+ }
+ }.resolve(this, absF);
}
/**
@@ -577,10 +599,15 @@
* @param f the file name to open
* @param bufferSize the size of the buffer to be used.
*/
- public FSDataInputStream open(final Path f, int bufferSize)
+ public FSDataInputStream open(final Path f, final int bufferSize)
throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).open(absF, bufferSize);
+ return new FSLinkResolver<FSDataInputStream>() {
+ public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.open(p, bufferSize);
+ }
+ }.resolve(this, absF);
}
/**
@@ -595,7 +622,12 @@
public boolean setReplication(final Path f, final short replication)
throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).setReplication(absF, replication);
+ return new FSLinkResolver<Boolean>() {
+ public Boolean next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return Boolean.valueOf(fs.setReplication(p, replication));
+ }
+ }.resolve(this, absF);
}
/**
@@ -633,7 +665,22 @@
if(!srcFS.getUri().equals(dstFS.getUri())) {
throw new IOException("Renames across AbstractFileSystems not supported");
}
- srcFS.rename(absSrc, absDst, options);
+ try {
+ srcFS.rename(absSrc, absDst, options);
+ } catch (UnresolvedLinkException e) {
+ /* We do not know whether the source or the destination path
+ * was unresolved. Resolve the source path completely, then
+ * resolve the destination.
+ */
+ final Path source = resolve(absSrc);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.rename(source, p, options);
+ return null;
+ }
+ }.resolve(this, absDst);
+ }
}
/**
@@ -644,7 +691,13 @@
public void setPermission(final Path f, final FsPermission permission)
throws IOException {
final Path absF = fixRelativePart(f);
- getFSofPath(absF).setPermission(absF, permission);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.setPermission(p, permission);
+ return null;
+ }
+ }.resolve(this, absF);
}
/**
@@ -655,13 +708,19 @@
* @param groupname If it is null, the original groupname remains unchanged.
*/
public void setOwner(final Path f, final String username,
- final String groupname) throws IOException {
+ final String groupname) throws IOException {
if ((username == null) && (groupname == null)) {
throw new IllegalArgumentException(
- "usernme and groupname cannot both be null");
+ "username and groupname cannot both be null");
}
final Path absF = fixRelativePart(f);
- getFSofPath(absF).setOwner(absF, username, groupname);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.setOwner(p, username, groupname);
+ return null;
+ }
+ }.resolve(this, absF);
}
/**
@@ -677,7 +736,13 @@
public void setTimes(final Path f, final long mtime, final long atime)
throws IOException {
final Path absF = fixRelativePart(f);
- getFSofPath(absF).setTimes(absF, mtime, atime);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.setTimes(p, mtime, atime);
+ return null;
+ }
+ }.resolve(this, absF);
}
/**
@@ -690,7 +755,12 @@
*/
public FileChecksum getFileChecksum(final Path f) throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).getFileChecksum(absF);
+ return new FSLinkResolver<FileChecksum>() {
+ public FileChecksum next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getFileChecksum(p);
+ }
+ }.resolve(this, absF);
}
/**
@@ -704,10 +774,8 @@
public void setVerifyChecksum(final boolean verifyChecksum, final Path f)
throws IOException {
- final Path absF = fixRelativePart(f);
+ final Path absF = resolve(fixRelativePart(f));
getFSofPath(absF).setVerifyChecksum(verifyChecksum);
-
- //TBD need to be changed when we add symlinks.
}
/**
@@ -719,7 +787,84 @@
*/
public FileStatus getFileStatus(final Path f) throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).getFileStatus(absF);
+ return new FSLinkResolver<FileStatus>() {
+ public FileStatus next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getFileStatus(p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Return a fully qualified version of the given symlink target if it
+ * has no scheme and authority. Partially and fully qualified paths
+ * are returned unmodified.
+ * @param linkFS The AbstractFileSystem of link
+ * @param link The path of the symlink
+ * @param target The symlink's target
+ * @return Fully qualified version of the target.
+ */
+ private Path qualifySymlinkTarget(final AbstractFileSystem linkFS,
+ Path link, Path target) {
+ /* NB: makeQualified uses link's scheme/authority, if specified,
+ * and the scheme/authority of linkFS, if not. If link does have
+ * a scheme and authority they should match those of linkFS since
+ * resolve updates the path and file system of a path that contains
+ * links each time a link is encountered.
+ */
+ final String linkScheme = link.toUri().getScheme();
+ final String linkAuth = link.toUri().getAuthority();
+ if (linkScheme != null && linkAuth != null) {
+ assert linkScheme.equals(linkFS.getUri().getScheme());
+ assert linkAuth.equals(linkFS.getUri().getAuthority());
+ }
+ final boolean justPath = (target.toUri().getScheme() == null &&
+ target.toUri().getAuthority() == null);
+ return justPath ? target.makeQualified(linkFS.getUri(), link.getParent())
+ : target;
+ }
+
+ /**
+ * Return a file status object that represents the path. If the path
+ * refers to a symlink then the FileStatus of the symlink is returned.
+ * The behavior is equivalent to #getFileStatus() if the underlying
+ * file system does not support symbolic links.
+ * @param f The path we want information from.
+ * @return A FileStatus object
+ * @throws FileNotFoundException when the path does not exist;
+ * IOException see specific implementation.
+ */
+ public FileStatus getFileLinkStatus(final Path f) throws IOException {
+ final Path absF = fixRelativePart(f);
+ return new FSLinkResolver<FileStatus>() {
+ public FileStatus next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ FileStatus fi = fs.getFileLinkStatus(p);
+ if (fi.isSymlink()) {
+ fi.setSymlink(qualifySymlinkTarget(fs, p, fi.getSymlink()));
+ }
+ return fi;
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Returns the un-interpreted target of the given symbolic link.
+ * Transparently resolves all links up to the final path component.
+ * @param f
+ * @return The un-interpreted target of the symbolic link.
+ * @throws FileNotFoundException when the path does not exist;
+ * IOException if the last path component of f is not a symlink.
+ */
+ public Path getLinkTarget(final Path f) throws IOException {
+ final Path absF = fixRelativePart(f);
+ return new FSLinkResolver<Path>() {
+ public Path next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ FileStatus fi = fs.getFileLinkStatus(p);
+ return fi.getSymlink();
+ }
+ }.resolve(this, absF);
}
/**
@@ -740,12 +885,18 @@
@InterfaceStability.Evolving
public BlockLocation[] getFileBlockLocations(final Path p,
final long start, final long len) throws IOException {
- return getFSofPath(p).getFileBlockLocations(p, start, len);
+ final Path absF = fixRelativePart(p);
+ return new FSLinkResolver<BlockLocation[]>() {
+ public BlockLocation[] next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getFileBlockLocations(p, start, len);
+ }
+ }.resolve(this, absF);
}
/**
* Returns a status object describing the use and capacity of the
- * filesystem denoted by the Parh argument p.
+ * filesystem denoted by the Path argument p.
* If the filesystem has multiple partitions, the
* use and capacity of the partition pointed to by the specified
* path is reflected.
@@ -758,12 +909,99 @@
*/
public FsStatus getFsStatus(final Path f) throws IOException {
if (f == null) {
- return defaultFS.getFsStatus(null);
+ return defaultFS.getFsStatus();
}
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).getFsStatus(absF);
+ return new FSLinkResolver<FsStatus>() {
+ public FsStatus next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getFsStatus(p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Creates a symbolic link to an existing file. An exception is thrown if
+ * the symlink exits, the user does not have permission to create symlink,
+ * or the underlying file system does not support symlinks.
+ *
+ * Symlink permissions are ignored, access to a symlink is determined by
+ * the permissions of the symlink target.
+ *
+ * Symlinks in paths leading up to the final path component are resolved
+ * transparently. If the final path component refers to a symlink some
+ * functions operate on the symlink itself, these are:
+ * - delete(f) and deleteOnExit(f) - Deletes the symlink.
+ * - rename(src, dst) - If src refers to a symlink, the symlink is
+ * renamed. If dst refers to a symlink, the symlink is over-written.
+ * - getLinkTarget(f) - Returns the target of the symlink.
+ * - getFileLinkStatus(f) - Returns a FileStatus object describing
+ * the symlink.
+ * Some functions, create() and mkdir(), expect the final path component
+ * does not exist. If they are given a path that refers to a symlink that
+ * does exist they behave as if the path referred to an existing file or
+ * directory. All other functions fully resolve, ie follow, the symlink.
+ * These are: open, setReplication, setOwner, setTimes, setWorkingDirectory,
+ * setPermission, getFileChecksum, setVerifyChecksum, getFileBlockLocations,
+ * getFsStatus, getFileStatus, isDirectory, isFile, exists, and listStatus.
+ *
+ * Symlink targets are stored as given to createSymlink, assuming the
+ * underlying file system is capable of storign a fully qualified URI.
+ * Dangling symlinks are permitted. FileContext supports four types of
+ * symlink targets, and resolves them as follows
+ * <pre>
+ * Given a path referring to a symlink of form:
+ *
+ * <---X--->
+ * fs://host/A/B/link
+ * <-----Y----->
+ *
+ * In this path X is the scheme and authority that identify the file system,
+ * and Y is the path leading up to the final path component "link". If Y is
+ * a symlink itself then let Y' be the target of Y and X' be the scheme and
+ * authority of Y'. Symlink targets may:
+ *
+ * 1. Fully qualified URIs
+ *
+ * fs://hostX/A/B/file Resolved according to the target file system.
+ *
+ * 2. Partially qualified URIs (eg scheme but no host)
+ *
+ * fs:///A/B/file Resolved according to the target file sytem. Eg resolving
+ * a symlink to hdfs:///A results in an exception because
+ * HDFS URIs must be fully qualified, while a symlink to
+ * file:///A will not since Hadoop's local file systems
+ * require partially qualified URIs.
+ *
+ * 3. Relative paths
+ *
+ * path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path
+ * is "../B/file" then [Y'][path] is hdfs://host/B/file
+ *
+ * 4. Absolute paths
+ *
+ * path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path
+ * is "/file" then [X][path] is hdfs://host/file
+ * </pre>
+ *
+ * @param target the target of the symbolic link
+ * @param link the path to be created that points to target
+ * @param createParent if true then missing parent dirs are created if
+ * false then parent must exist
+ * @throws IOException
+ */
+ public void createSymlink(final Path target, final Path link,
+ final boolean createParent) throws IOException {
+ final Path nonRelLink = fixRelativePart(link);
+ new FSLinkResolver<Void>() {
+ public Void next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ fs.createSymlink(target, p, createParent);
+ return null;
+ }
+ }.resolve(this, nonRelLink);
}
-
+
/**
* Does the file exist?
* Note: Avoid using this method if you already have FileStatus in hand.
@@ -821,7 +1059,12 @@
*/
public FileStatus[] listStatus(final Path f) throws IOException {
final Path absF = fixRelativePart(f);
- return getFSofPath(absF).listStatus(absF);
+ return new FSLinkResolver<FileStatus[]>() {
+ public FileStatus[] next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.listStatus(p);
+ }
+ }.resolve(this, absF);
}
/**
@@ -920,8 +1163,7 @@
* applying the filter default Path filter
* @exception IOException
*/
- public FileStatus[] listStatus(Path[] files)
- throws IOException {
+ public FileStatus[] listStatus(Path[] files) throws IOException {
return listStatus(files, DEFAULT_FILTER);
}
@@ -1264,9 +1506,6 @@
/** Default pattern character: Character set close. */
private static final char PAT_SET_CLOSE = ']';
- GlobFilter() {
- }
-
GlobFilter(final String filePattern) throws IOException {
setRegex(filePattern);
}
@@ -1441,4 +1680,73 @@
processDeleteOnExit();
}
}
-}
+
+ /**
+ * Resolves all symbolic links in the specified path.
+ * Returns the new path object.
+ */
+ protected Path resolve(final Path f) throws IOException {
+ return new FSLinkResolver<FileStatus>() {
+ public FileStatus next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException {
+ return fs.getFileStatus(p);
+ }
+ }.resolve(this, f).getPath();
+ }
+
+ /**
+ * Class used to perform an operation on and resolve symlinks in a
+ * path. The operation may potentially span multiple file systems.
+ */
+ protected abstract class FSLinkResolver<T> {
+ // The maximum number of symbolic link components in a path
+ private static final int MAX_PATH_LINKS = 32;
+
+ /**
+ * Generic helper function overridden on instantiation to perform a
+ * specific operation on the given file system using the given path
+ * which may result in an UnresolvedLinkException.
+ * @param fs AbstractFileSystem to perform the operation on.
+ * @param p Path given the file system.
+ * @return Generic type determined by the specific implementation.
+ * @throws IOException on error.
+ * @throws UnresolvedLinkException when a symlink is encountered.
+ */
+ public abstract T next(final AbstractFileSystem fs, final Path p)
+ throws IOException, UnresolvedLinkException;
+
+ /**
+ * Performs the operation specified by the next function, calling it
+ * repeatedly until all symlinks in the given path are resolved.
+ * @param fc FileContext used to access file systems.
+ * @param p The path to resolve symlinks in.
+ * @return Generic type determined by the implementation of next.
+ * @throws IOException
+ */
+ public T resolve(final FileContext fc, Path p) throws IOException {
+ int count = 0;
+ T in = null;
+ Path first = p;
+ // NB: More than one AbstractFileSystem can match a scheme, eg
+ // "file" resolves to LocalFs but could have come by RawLocalFs.
+ AbstractFileSystem fs = fc.getFSofPath(p);
+
+ // Loop until all symlinks are resolved or the limit is reached
+ for (boolean isLink = true; isLink;) {
+ try {
+ in = next(fs, p);
+ isLink = false;
+ } catch (UnresolvedLinkException e) {
+ if (count++ > MAX_PATH_LINKS) {
+ throw new IOException("Possible cyclic loop while " +
+ "following symbolic link " + first);
+ }
+ // Resolve the first unresolved path component
+ p = qualifySymlinkTarget(fs, p, fs.getLinkTarget(p));
+ fs = fc.getFSofPath(p);
+ }
+ }
+ return in;
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileStatus.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileStatus.java Tue Feb 16 21:43:30 2010
@@ -39,6 +39,7 @@
private FsPermission permission;
private String owner;
private String group;
+ private Path symlink;
public FileStatus() { this(0, false, 0, 0, 0, 0, null, null, null, null); }
@@ -49,10 +50,24 @@
this(length, isdir, block_replication, blocksize, modification_time,
0, null, null, null, path);
}
-
- public FileStatus(long length, boolean isdir, int block_replication,
+
+ /**
+ * Constructor for file systems on which symbolic links are not supported
+ */
+ public FileStatus(long length, boolean isdir,
+ int block_replication,
+ long blocksize, long modification_time, long access_time,
+ FsPermission permission, String owner, String group,
+ Path path) {
+ this(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, owner, group, null, path);
+ }
+
+ public FileStatus(long length, boolean isdir,
+ int block_replication,
long blocksize, long modification_time, long access_time,
FsPermission permission, String owner, String group,
+ Path symlink,
Path path) {
this.length = length;
this.isdir = isdir;
@@ -64,6 +79,7 @@
FsPermission.getDefault() : permission;
this.owner = (owner == null) ? "" : owner;
this.group = (group == null) ? "" : group;
+ this.symlink = symlink;
this.path = path;
}
@@ -182,6 +198,28 @@
this.group = (group == null) ? "" : group;
}
+ /**
+ * Is this a symbolic link?
+ * @return true if this is a symbolic link
+ */
+ public boolean isSymlink() {
+ return symlink != null;
+ }
+
+ /**
+ * @return The contents of the symbolic link.
+ */
+ public Path getSymlink() throws IOException {
+ if (!isSymlink()) {
+ throw new IOException("Path " + path + " is not a symbolic link");
+ }
+ return symlink;
+ }
+
+ public void setSymlink(final Path p) {
+ symlink = p;
+ }
+
//////////////////////////////////////////////////
// Writable
//////////////////////////////////////////////////
@@ -196,6 +234,10 @@
permission.write(out);
Text.writeString(out, owner);
Text.writeString(out, group);
+ out.writeBoolean(isSymlink());
+ if (isSymlink()) {
+ Text.writeString(out, symlink.toString());
+ }
}
public void readFields(DataInput in) throws IOException {
@@ -210,6 +252,11 @@
permission.readFields(in);
owner = Text.readString(in);
group = Text.readString(in);
+ if (in.readBoolean()) {
+ this.symlink = new Path(Text.readString(in));
+ } else {
+ this.symlink = null;
+ }
}
/**
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FilterFs.java Tue Feb 16 21:43:30 2010
@@ -60,38 +60,49 @@
protected FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
short replication, long blockSize, Progressable progress,
- int bytesPerChecksum, boolean createParent) throws IOException {
+ int bytesPerChecksum, boolean createParent)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.createInternal(f, flag, absolutePermission, bufferSize,
replication, blockSize, progress, bytesPerChecksum, createParent);
}
@Override
- protected boolean delete(Path f, boolean recursive) throws IOException {
+ protected boolean delete(Path f, boolean recursive)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.delete(f, recursive);
}
@Override
protected BlockLocation[] getFileBlockLocations(Path f, long start, long len)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.getFileBlockLocations(f, start, len);
}
@Override
- protected FileChecksum getFileChecksum(Path f) throws IOException {
+ protected FileChecksum getFileChecksum(Path f)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.getFileChecksum(f);
}
@Override
- protected FileStatus getFileStatus(Path f) throws IOException {
+ protected FileStatus getFileStatus(Path f)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.getFileStatus(f);
}
@Override
+ protected FileStatus getFileLinkStatus(final Path f)
+ throws IOException, UnresolvedLinkException {
+ checkPath(f);
+ return myFs.getFileLinkStatus(f);
+ }
+
+ @Override
protected FsStatus getFsStatus() throws IOException {
return myFs.getFsStatus();
}
@@ -107,36 +118,38 @@
}
@Override
- protected FileStatus[] listStatus(Path f) throws IOException {
+ protected FileStatus[] listStatus(Path f)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.listStatus(f);
}
@Override
protected void mkdir(Path dir, FsPermission permission, boolean createParent)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(dir);
myFs.mkdir(dir, permission, createParent);
}
@Override
- protected FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ protected FSDataInputStream open(Path f, int bufferSize)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.open(f, bufferSize);
}
@Override
- protected void renameInternal(Path src, Path dst) throws IOException {
+ protected void renameInternal(Path src, Path dst)
+ throws IOException, UnresolvedLinkException {
checkPath(src);
checkPath(dst);
myFs.rename(src, dst, Options.Rename.NONE);
-
}
@Override
protected void setOwner(Path f, String username, String groupname)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(f);
myFs.setOwner(f, username, groupname);
@@ -144,27 +157,44 @@
@Override
protected void setPermission(Path f, FsPermission permission)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(f);
myFs.setPermission(f, permission);
}
@Override
protected boolean setReplication(Path f, short replication)
- throws IOException {
+ throws IOException, UnresolvedLinkException {
checkPath(f);
return myFs.setReplication(f, replication);
}
@Override
- protected void setTimes(Path f, long mtime, long atime) throws IOException {
+ protected void setTimes(Path f, long mtime, long atime)
+ throws IOException, UnresolvedLinkException {
checkPath(f);
myFs.setTimes(f, mtime, atime);
-
}
@Override
- protected void setVerifyChecksum(boolean verifyChecksum) throws IOException {
+ protected void setVerifyChecksum(boolean verifyChecksum)
+ throws IOException, UnresolvedLinkException {
myFs.setVerifyChecksum(verifyChecksum);
}
+
+ @Override
+ protected boolean supportsSymlinks() {
+ return myFs.supportsSymlinks();
+ }
+
+ @Override
+ protected void createSymlink(Path target, Path link, boolean createParent)
+ throws IOException, UnresolvedLinkException {
+ myFs.createSymlink(target, link, createParent);
+ }
+
+ @Override
+ protected Path getLinkTarget(final Path f) throws IOException {
+ return myFs.getLinkTarget(f);
+ }
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/Path.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/Path.java Tue Feb 16 21:43:30 2010
@@ -191,7 +191,7 @@
return uri.getPath().startsWith(SEPARATOR, start);
}
- /** True if the directory of this path is absolute. */
+ /** True if the path component of this URI is absolute. */
/**
* There is some ambiguity here. An absolute path is a slash
* relative name without a scheme or an authority.
Added: hadoop/common/trunk/src/java/org/apache/hadoop/fs/UnresolvedLinkException.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/UnresolvedLinkException.java?rev=910706&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/UnresolvedLinkException.java (added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/UnresolvedLinkException.java Tue Feb 16 21:43:30 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate.*;
+
+/**
+ * Thrown when a symbolic link is encountered in a path.
+ */
+@InterfaceAudience.LimitedPrivate({Project.HDFS})
+public class UnresolvedLinkException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ public UnresolvedLinkException() {
+ super();
+ }
+
+ public UnresolvedLinkException(String msg) {
+ super(msg);
+ }
+}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/local/RawLocalFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/local/RawLocalFs.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/local/RawLocalFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/local/RawLocalFs.java Tue Feb 16 21:43:30 2010
@@ -18,17 +18,21 @@
package org.apache.hadoop.fs.local;
import java.io.IOException;
+import java.io.FileNotFoundException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.RawLocalFileSystem;
-
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
/**
* The RawLocalFs implementation of AbstractFileSystem.
@@ -37,6 +41,7 @@
@InterfaceAudience.Private
@InterfaceStability.Evolving /*Evolving for a release,to be changed to Stable */
public class RawLocalFs extends DelegateToFileSystem {
+
RawLocalFs(final Configuration conf) throws IOException, URISyntaxException {
this(FsConstants.LOCAL_FS_URI, conf);
}
@@ -65,4 +70,102 @@
protected FsServerDefaults getServerDefaults() throws IOException {
return LocalConfigKeys.getServerDefaults();
}
+
+ @Override
+ protected boolean supportsSymlinks() {
+ return true;
+ }
+
+ @Override
+ protected void createSymlink(Path target, Path link, boolean createParent)
+ throws IOException {
+ final String targetScheme = target.toUri().getScheme();
+ if (targetScheme != null && !"file".equals(targetScheme)) {
+ throw new IOException("Unable to create symlink to non-local file "+
+ "system: "+target.toString());
+ }
+ if (createParent) {
+ mkdir(link.getParent(), FsPermission.getDefault(), true);
+ }
+ // NB: Use createSymbolicLink in java.nio.file.Path once available
+ try {
+ Shell.execCommand(Shell.LINK_COMMAND, "-s",
+ new URI(target.toString()).getPath(),
+ new URI(link.toString()).getPath());
+ } catch (URISyntaxException x) {
+ throw new IOException("Invalid symlink path: "+x.getMessage());
+ } catch (IOException x) {
+ throw new IOException("Unable to create symlink: "+x.getMessage());
+ }
+ }
+
+ /**
+ * Returns the target of the given symlink. Returns the empty string if
+ * the given path does not refer to a symlink or there is an error
+ * acessing the symlink.
+ */
+ private String readLink(Path p) {
+ /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could
+ * use getCanonicalPath in File to get the target of the symlink but that
+ * does not indicate if the given path refers to a symlink.
+ */
+ try {
+ final String path = p.toUri().getPath();
+ return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim();
+ } catch (IOException x) {
+ return "";
+ }
+ }
+
+ /**
+ * Return a FileStatus representing the given path. If the path refers
+ * to a symlink return a FileStatus representing the link rather than
+ * the object the link refers to.
+ */
+ @Override
+ protected FileStatus getFileLinkStatus(final Path f) throws IOException {
+ String target = readLink(f);
+ try {
+ FileStatus fs = getFileStatus(f);
+ // If f refers to a regular file or directory
+ if ("".equals(target)) {
+ return fs;
+ }
+ // Otherwise f refers to a symlink
+ return new FileStatus(fs.getLen(),
+ false,
+ fs.getReplication(),
+ fs.getBlockSize(),
+ fs.getModificationTime(),
+ fs.getAccessTime(),
+ fs.getPermission(),
+ fs.getOwner(),
+ fs.getGroup(),
+ new Path(target),
+ f);
+ } catch (FileNotFoundException e) {
+ /* The exists method in the File class returns false for dangling
+ * links so we can get a FileNotFoundException for links that exist.
+ * It's also possible that we raced with a delete of the link. Use
+ * the readBasicFileAttributes method in java.nio.file.attributes
+ * when available.
+ */
+ if (!"".equals(target)) {
+ return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(),
+ "", "", new Path(target), f);
+ }
+ // f refers to a file or directory that does not exist
+ throw e;
+ }
+ }
+
+ @Override
+ protected Path getLinkTarget(Path f) throws IOException {
+ /* We should never get here. Valid local links are resolved transparently
+ * by the underlying local file system and accessing a dangling link will
+ * result in an IOException, not an UnresolvedLinkException, so FileContext
+ * should never call this function.
+ */
+ throw new AssertionError();
+ }
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/util/Shell.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/util/Shell.java Tue Feb 16 21:43:30 2010
@@ -57,6 +57,10 @@
/** a Unix command to set owner */
public static final String SET_OWNER_COMMAND = "chown";
public static final String SET_GROUP_COMMAND = "chgrp";
+ /** a Unix command to create a link */
+ public static final String LINK_COMMAND = "ln";
+ /** a Unix command to get a link target */
+ public static final String READ_LINK_COMMAND = "readlink";
/** Return a Unix command to get permission information. */
public static String[] getGET_PERMISSION_COMMAND() {
//force /bin/ls, except on windows.
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java?rev=910706&r1=910705&r2=910706&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java Tue Feb 16 21:43:30 2010
@@ -967,6 +967,31 @@
out.close();
}
+ @Test
+ /** Test FileContext APIs when symlinks are not supported */
+ public void testUnsupportedSymlink() throws IOException {
+ Path file = getTestRootPath(fc, "file");
+ Path link = getTestRootPath(fc, "linkToFile");
+ if (!fc.getDefaultFileSystem().supportsSymlinks()) {
+ try {
+ fc.createSymlink(file, link, false);
+ Assert.fail("Created a symlink on a file system that "+
+ "does not support symlinks.");
+ } catch (IOException e) {
+ // Expected
+ }
+ createFile(file);
+ try {
+ fc.getLinkTarget(file);
+ Assert.fail("Got a link target on a file system that "+
+ "does not support symlinks.");
+ } catch (IOException e) {
+ // Expected
+ }
+ Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file));
+ }
+ }
+
protected void createFile(Path path) throws IOException {
FSDataOutputStream out = fc.create(path, EnumSet.of(CreateFlag.CREATE),
Options.CreateOpts.createParent());