You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2008/01/26 00:15:17 UTC
svn commit: r615392 - in /hadoop/core/trunk: ./
src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/kfs/ src/java/org/apache/hadoop/fs/s3/
Author: hairong
Date: Fri Jan 25 15:15:15 2008
New Revision: 615392
URL: http://svn.apache.org/viewvc?rev=615392&view=rev
Log:
HADOOP-2566. Add globStatus method to the FileSystem interface and deprecate globPath and listPath.(Hairong Kuang via hairong)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileStatus.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri Jan 25 15:15:15 2008
@@ -782,6 +782,9 @@
HADOOP-1968. FileSystem supports wildcard input syntax "{ }".
(Hairong Kuang via dhruba)
+ HADOOP-2566. Add globStatus method to the FileSystem interface
+ and deprecate globPath and listPath. (Hairong Kuang via hairong)
+
OPTIMIZATIONS
HADOOP-1910. Reduce the number of RPCs that DistributedFileSystem.create()
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Fri Jan 25 15:15:15 2008
@@ -23,6 +23,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.dfs.FSConstants.DatanodeReportType;
import org.apache.hadoop.dfs.FSConstants.UpgradeAction;
@@ -321,15 +322,26 @@
/**
* Returns the stat information about the file.
+ * @throws FileNotFoundException if the file does not exist.
*/
public FileStatus getFileStatus(Path f) throws IOException {
if (f instanceof DfsPath) {
DfsPath p = (DfsPath) f;
return p.info;
}
- else {
+
+ try {
DFSFileInfo p = dfs.getFileInfo(getPathName(f));
return p;
+ } catch (RemoteException e) {
+ if (IOException.class.getName().equals(e.getClassName()) &&
+ e.getMessage().startsWith(
+ "java.io.IOException: File does not exist: ")) {
+ // non-existent path
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ } else {
+ throw e; // unexpected exception
+ }
}
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/HftpFileSystem.java Fri Jan 25 15:15:15 2008
@@ -18,6 +18,7 @@
package org.apache.hadoop.dfs;
+import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
@@ -192,7 +193,7 @@
public FileStatus getFileStatus(Path f) throws IOException {
fetchList(f.toUri().getPath(), false);
if (fslist.size() == 0) {
- throw new IOException("File does not exist: " + f);
+ throw new FileNotFoundException("File does not exist: " + f);
}
return fslist.get(0);
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileStatus.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileStatus.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileStatus.java Fri Jan 25 15:15:15 2008
@@ -27,7 +27,7 @@
/** Interface that represents the client side information for a file.
*/
-public class FileStatus implements Writable {
+public class FileStatus implements Writable, Comparable {
private Path path;
private long length;
@@ -195,4 +195,46 @@
group = Text.readString(in);
}
+ /**
+ * Compare this object to another object
+ *
+ * @param o the object to be compared.
+ * @return a negative integer, zero, or a positive integer as this object
+ * is less than, equal to, or greater than the specified object.
+ *
+ * @throws ClassCastException if the specified object's is not of
+ * type FileStatus
+ */
+ public int compareTo(Object o) {
+ FileStatus other = (FileStatus)o;
+ return this.getPath().compareTo(other.getPath());
+ }
+
+ /** Compare if this object is equal to another object
+ * @param o the object to be compared.
+ * @return true if two file status has the same path name; false if not.
+ */
+ public boolean equals(Object o) {
+ if (o == null) {
+ return false;
+ }
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof FileStatus)) {
+ return false;
+ }
+ FileStatus other = (FileStatus)o;
+ return this.getPath().equals(other.getPath());
+ }
+
+ /**
+ * Returns a hash code value for the object, which is defined as
+ * the hash code of the path name.
+ *
+ * @return a hash code value for the path name.
+ */
+ public int hashCode() {
+ return getPath().hashCode();
+ }
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Fri Jan 25 15:15:15 2008
@@ -518,21 +518,17 @@
public boolean isDirectory(Path f) throws IOException {
try {
return getFileStatus(f).isDir();
- } catch (IOException e) {
- if (e instanceof RemoteException && AccessControlException.class
- .getName().equals(((RemoteException)e).getClassName())) {
- throw e;
- }
+ } catch (FileNotFoundException e) {
return false; // f does not exist
}
}
/** True iff the named path is a regular file. */
public boolean isFile(Path f) throws IOException {
- if (exists(f) && !isDirectory(f)) {
- return true;
- } else {
- return false;
+ try {
+ return !getFileStatus(f).isDir();
+ } catch (FileNotFoundException e) {
+ return false; // f does not exist
}
}
@@ -579,37 +575,88 @@
return ret;
}
- /** */
+ /**
+ * List the statuses of the files/directories in the given path if the path is
+ * a directory.
+ *
+ * @param f
+ * given path
+ * @return the statuses of the files/directories in the given patch
+ * @throws IOException
+ */
public abstract FileStatus[] listStatus(Path f) throws IOException;
+ /*
+ * Filter files/directories in the given path using the user-supplied path
+ * filter. Results are added to the given array <code>results</code>.
+ */
+ private void listStatus(ArrayList<FileStatus> results, Path f,
+ PathFilter filter) throws IOException {
+ FileStatus listing[] = listStatus(f);
+ if (listing != null) {
+ for (int i = 0; i < listing.length; i++) {
+ if (filter.accept(listing[i].getPath())) {
+ results.add(listing[i]);
+ }
+ }
+ }
+ }
+
+ /**
+ * Filter files/directories in the given path using the user-supplied path
+ * filter.
+ *
+ * @param f
+ * a path name
+ * @param filter
+ * the user-supplied path filter
+ * @return an array of FileStatus objects for the files under the given path
+ * after applying the filter
+ * @throws IOException
+ * if encounter any problem while fetching the status
+ */
+ public FileStatus[] listStatus(Path f, PathFilter filter) throws IOException {
+ ArrayList<FileStatus> results = new ArrayList<FileStatus>();
+ listStatus(results, f, filter);
+ return results.toArray(new FileStatus[results.size()]);
+ }
+
+ /**
+ * Filter files/directories in the given list of paths using user-supplied
+ * path filter.
+ *
+ * @param files
+ * a list of paths
+ * @param filter
+ * the user-supplied path filter
+ * @return a list of statuses for the files under the given paths after
+ * applying the filter
+ * @exception IOException
+ */
+ private FileStatus[] listStatus(Path[] files, PathFilter filter)
+ throws IOException {
+ ArrayList<FileStatus> results = new ArrayList<FileStatus>();
+ for (int i = 0; i < files.length; i++) {
+ listStatus(results, files[i], filter);
+ }
+ return results.toArray(new FileStatus[results.size()]);
+ }
+
/**
* Filter files in the given pathes using the default checksum filter.
* @param files a list of paths
* @return a list of files under the source paths
* @exception IOException
*/
+ @Deprecated
public Path[] listPaths(Path[] files) throws IOException {
return listPaths(files, DEFAULT_FILTER);
}
/** Filter files in a directory. */
- private void listPaths(ArrayList<Path> results, Path f, PathFilter filter)
- throws IOException {
- Path listing[] = listPaths(f);
- if (listing != null) {
- for (int i = 0; i < listing.length; i++) {
- if (filter.accept(listing[i])) {
- results.add(listing[i]);
- }
- }
- }
- }
-
- /** Filter files in a directory. */
+ @Deprecated
public Path[] listPaths(Path f, PathFilter filter) throws IOException {
- ArrayList<Path> results = new ArrayList<Path>();
- listPaths(results, f, filter);
- return results.toArray(new Path[results.size()]);
+ return FileUtil.stat2Paths(listStatus(f, filter));
}
/**
@@ -618,13 +665,10 @@
* @return a list of files under the source paths
* @exception IOException
*/
+ @Deprecated
public Path[] listPaths(Path[] files, PathFilter filter)
throws IOException {
- ArrayList<Path> results = new ArrayList<Path>();
- for(int i=0; i<files.length; i++) {
- listPaths(results, files[i], filter);
- }
- return results.toArray(new Path[results.size()]);
+ return FileUtil.stat2Paths(listStatus(files, filter));
}
/**
@@ -679,66 +723,138 @@
* </dd>
* </dl>
*
- * @param filePattern a regular expression specifying file pattern
+ * @param pathPattern a regular expression specifying a pth pattern
- * @return an array of paths that match the file pattern
+ * @return an array of paths that match the path pattern
* @throws IOException
*/
- public Path[] globPaths(Path filePattern) throws IOException {
- return globPaths(filePattern, DEFAULT_FILTER);
+ public FileStatus[] globStatus(Path pathPattern) throws IOException {
+ return globStatus(pathPattern, DEFAULT_FILTER);
}
-
- /** glob all the file names that matches filePattern
- * and is accepted by filter.
- */
- public Path[] globPaths(Path filePattern, PathFilter filter)
- throws IOException {
- Path [] parents = new Path[1];
+
+ /**
+ * Return an array of FileStatus objects whose path names match pathPattern
+ * and is accepted by the user-supplied path filter. Results are sorted by
+ * their path names.
+ * Return null if pathPattern has no glob and the path does not exist.
+ * Return an empty array if pathPattern has a glob and no path matches it.
+ *
+ * @param pathPattern
+ * a regular expression specifying the path pattern
+ * @param filter
+ * a user-supplied path filter
+ * @return an array of FileStatus objects
+ * @throws IOException if any I/O error occurs when fetching file status
+ */
+ public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
+ throws IOException {
+ Path[] parents = new Path[1];
int level = 0;
- String filename = filePattern.toUri().getPath();
+ String filename = pathPattern.toUri().getPath();
+
+ // path has only zero component
if ("".equals(filename) || Path.SEPARATOR.equals(filename)) {
- parents[0] = filePattern;
- return parents;
+ return getFileStatus(new Path[]{pathPattern});
}
-
- String [] components = filename.split(Path.SEPARATOR);
- if (filePattern.isAbsolute()) {
+
+ // path has at least one component
+ String[] components = filename.split(Path.SEPARATOR);
+ // get the first component
+ if (pathPattern.isAbsolute()) {
parents[0] = new Path(Path.SEPARATOR);
level = 1;
} else {
parents[0] = new Path(Path.CUR_DIR);
}
-
- Path[] results = globPathsLevel(parents, components, level, filter, false);
- Arrays.sort(results);
+
+ // glob the paths that match the parent path, i.e., [0, components.length-1]
+ boolean[] hasGlob = new boolean[]{false};
+ Path[] parentPaths =
+ globPathsLevel(parents, components, level, filter, hasGlob);
+ FileStatus[] results;
+ if (parentPaths == null || parentPaths.length == 0) {
+ results = null;
+ } else {
+ // Now work on the last component of the path
+ GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
+ if (fp.hasPattern()) { // last component has a pattern
+ // list parent directories and then glob the results
+ results = listStatus(parentPaths, fp);
+ hasGlob[0] = true;
+ } else { // last component does not have a pattern
+ // get all the path names
+ for (int i = 0; i < parentPaths.length; i++) {
+ parentPaths[i] = new Path(parentPaths[i],
+ components[components.length - 1]);
+ }
+ // get all their statuses
+ results = getFileStatus(parentPaths);
+ }
+ }
+
+ // Decide if the pathPattern contains a glob or not
+ if (results == null) {
+ if (hasGlob[0]) {
+ results = new FileStatus[0];
+ }
+ } else {
+ if (results.length == 0 ) {
+ if (!hasGlob[0]) {
+ results = null;
+ }
+ } else {
+ Arrays.sort(results);
+ }
+ }
return results;
}
-
- private Path[] globPathsLevel(Path[] parents, String [] filePattern,
- int level, PathFilter filter, boolean hasGlob) throws IOException {
- if (level == filePattern.length)
+
+ /**
+ * glob all the path names that match filePattern using the default filter
+ */
+ @Deprecated
+ public Path[] globPaths(Path filePattern) throws IOException {
+ return globPaths(filePattern, DEFAULT_FILTER);
+ }
+
+ /**
+ * glob all the path names that match filePattern and is accepted by filter.
+ */
+ @Deprecated
+ public Path[] globPaths(Path filePattern, PathFilter filter)
+ throws IOException {
+ FileStatus[] stats = globStatus(filePattern, filter);
+ if (stats == null) {
+ return new Path[]{filePattern};
+ } else {
+ return FileUtil.stat2Paths(stats);
+ }
+ }
+
+ /*
+ * For a path of N components, return a list of paths that match the
+ * components [<code>level</code>, <code>N-1</code>].
+ */
+ private Path[] globPathsLevel(Path[] parents, String[] filePattern,
+ int level, PathFilter filter, boolean[] hasGlob) throws IOException {
+ if (level == filePattern.length - 1)
return parents;
+ if (parents == null || parents.length == 0) {
+ return null;
+ }
GlobFilter fp = new GlobFilter(filePattern[level], filter);
if (fp.hasPattern()) {
- parents = listPaths(parents, fp);
- hasGlob = true;
+ parents = FileUtil.stat2Paths(listStatus(parents, fp));
+ hasGlob[0] = true;
} else {
- int goodPathCount = 0;
- for(int i=0; i<parents.length; i++) {
- Path tmpPath = new Path(parents[i], filePattern[level]);
- if (!hasGlob || exists(tmpPath)) {
- parents[goodPathCount++] = tmpPath;
- }
- }
- if (goodPathCount != parents.length) {
- Path [] goodParents = new Path[goodPathCount];
- System.arraycopy(parents, 0, goodParents, 0, goodPathCount);
- parents = goodParents;
+ for (int i = 0; i < parents.length; i++) {
+ parents[i] = new Path(parents[i], filePattern[level]);
}
}
- return globPathsLevel(parents, filePattern, level+1, filter, hasGlob);
+ return globPathsLevel(parents, filePattern, level + 1, filter, hasGlob);
}
-
+
+ /* A class that could decide if a string matches the glob or not */
private static class GlobFilter implements PathFilter {
private PathFilter userFilter = DEFAULT_FILTER;
private Pattern regex;
@@ -1052,13 +1168,37 @@
public short getDefaultReplication() { return 1; }
/**
- * Return a file status object that represents the
- * file.
- * @param f The path to the file we want information from
- * @return filestatus object
- * @throws IOException see specific implementation
+ * Return a file status object that represents the path.
+ * @param f The path we want information from
+ * @return a FileStatus object
+ * @throws FileNotFoundException when the path does not exist;
+ * IOException see specific implementation
*/
public abstract FileStatus getFileStatus(Path f) throws IOException;
+
+ /**
+ * Return a list of file status objects that corresponds to the list of paths
+ * excluding those non-existent paths.
+ *
+ * @param paths
+ * the list of paths we want information from
+ * @return a list of FileStatus objects
+ * @throws IOException
+ * see specific implementation
+ */
+ private FileStatus[] getFileStatus(Path[] paths) throws IOException {
+ if (paths == null) {
+ return null;
+ }
+ ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
+ for (int i = 0; i < paths.length; i++) {
+ try {
+ results.add(getFileStatus(paths[i]));
+ } catch (FileNotFoundException e) { // do nothing
+ }
+ }
+ return results.toArray(new FileStatus[results.size()]);
+ }
/**
* Set permission of a path.
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java Fri Jan 25 15:15:15 2008
@@ -31,7 +31,23 @@
* A collection of file-processing util methods
*/
public class FileUtil {
-
+ /**
+ * convert an array of FileStatus to an array of Path
+ *
+ * @param stats
+ * an array of FileStatus objects
+ * @return an array of paths corresponding to the input
+ */
+ public static Path[] stat2Paths(FileStatus[] stats) {
+ if (stats == null)
+ return null;
+ Path[] ret = new Path[stats.length];
+ for (int i = 0; i < stats.length; ++i) {
+ ret[i] = stats[i].getPath();
+ }
+ return ret;
+ }
+
/**
* Delete a directory and all its contents. If
* we return false, the directory may be partially-deleted.
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java Fri Jan 25 15:15:15 2008
@@ -288,7 +288,11 @@
public FileStatus getFileStatus(Path f) throws IOException {
synchronized (this) {
- return new InMemoryFileStatus(f, pathToFileAttribs.get(getPath(f)));
+ FileAttributes attr = pathToFileAttribs.get(getPath(f));
+ if (attr==null) {
+ throw new FileNotFoundException("File " + f + " does not exist.");
+ }
+ return new InMemoryFileStatus(f, attr);
}
}
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Fri Jan 25 15:15:15 2008
@@ -359,8 +359,13 @@
return "LocalFS";
}
- public FileStatus getFileStatus(Path f) {
- return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
+ public FileStatus getFileStatus(Path f) throws IOException {
+ File path = pathToFile(f);
+ if (path.exists()) {
+ return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this);
+ } else {
+ throw new FileNotFoundException( "File " + f + " does not exist.");
+ }
}
static class RawLocalFileStatus extends FileStatus {
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/kfs/KosmosFileSystem.java Fri Jan 25 15:15:15 2008
@@ -196,7 +196,9 @@
public FileStatus getFileStatus(Path path) throws IOException {
Path absolute = makeAbsolute(path);
String srep = absolute.toUri().getPath();
-
+ if (!kfsImpl.exists(srep)) {
+ throw new FileNotFoundException("File " + path + " does not exist.");
+ }
if (kfsImpl.isDirectory(srep)) {
// System.out.println("Status of path: " + path + " is dir");
return new FileStatus(0, true, 1, 0, 0, path);
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java?rev=615392&r1=615391&r2=615392&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/s3/S3FileSystem.java Fri Jan 25 15:15:15 2008
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -298,7 +299,7 @@
public FileStatus getFileStatus(Path f) throws IOException {
INode inode = store.retrieveINode(makeAbsolute(f));
if (inode == null) {
- throw new IOException(f.toString() + ": No such file or directory.");
+ throw new FileNotFoundException(f + ": No such file or directory.");
}
return new S3FileStatus(f, inode);
}