You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 05:41:03 UTC
svn commit: r1079162 - in
/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs:
LocalDirAllocator.java LocalFileSystem.java Path.java
Author: omalley
Date: Tue Mar 8 04:41:03 2011
New Revision: 1079162
URL: http://svn.apache.org/viewvc?rev=1079162&view=rev
Log:
commit 7322dbc945d42b1276f8bbb657c1021014d63e58
Author: Rajesh Balamohan <rb...@yahoo-inc.com>
Date: Fri Jan 21 09:54:14 2011 +0530
. Reducing new Path(), RawFileStatus() creation overhead in LocalDirAllocator
Modified:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalFileSystem.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/Path.java
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalDirAllocator.java?rev=1079162&r1=1079161&r2=1079162&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalDirAllocator.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalDirAllocator.java Tue Mar 8 04:41:03 2011
@@ -239,7 +239,7 @@ public class LocalDirAllocator {
private FileSystem localFS;
private DF[] dirDF;
private String contextCfgItemName;
- private String[] localDirs;
+ private Path[] localDirsPath;
private String savedLocalDirs = "";
public AllocatorPerContext(String contextCfgItemName) {
@@ -253,7 +253,7 @@ public class LocalDirAllocator {
) throws IOException {
String newLocalDirs = conf.get(contextCfgItemName);
if (!newLocalDirs.equals(savedLocalDirs)) {
- localDirs = conf.getTrimmedStrings(contextCfgItemName);
+ String[] localDirs = conf.getTrimmedStrings(contextCfgItemName);
localFS = FileSystem.getLocal(conf);
int numDirs = localDirs.length;
ArrayList<String> dirs = new ArrayList<String>(numDirs);
@@ -279,7 +279,10 @@ public class LocalDirAllocator {
ie.getMessage() + "\n" + StringUtils.stringifyException(ie));
} //ignore
}
- localDirs = dirs.toArray(new String[dirs.size()]);
+ localDirsPath = new Path[dirs.size()];
+ for(int i=0;i<localDirsPath.length;i++) {
+ localDirsPath[i] = new Path(dirs.get(i));
+ }
dirDF = dfList.toArray(new DF[dirs.size()]);
savedLocalDirs = newLocalDirs;
@@ -288,10 +291,10 @@ public class LocalDirAllocator {
}
}
- private Path createPath(String path,
+ private Path createPath(Path path,
boolean checkWrite) throws IOException {
- Path file = new Path(new Path(localDirs[dirNumLastAccessed]),
- path);
+ Path file = new Path(localDirsPath[dirNumLastAccessed], path);
+
if (checkWrite) {
//check whether we are able to create a directory here. If the disk
//happens to be RDONLY we will fail
@@ -325,7 +328,7 @@ public class LocalDirAllocator {
Configuration conf, boolean checkWrite
) throws IOException {
confChanged(conf);
- int numDirs = localDirs.length;
+ int numDirs = localDirsPath.length;
int numDirsSearched = 0;
//remove the leading slash from the path (to make sure that the uri
//resolution results in a valid path on the dir being checked)
@@ -333,6 +336,7 @@ public class LocalDirAllocator {
pathStr = pathStr.substring(1);
}
Path returnPath = null;
+ Path path = new Path(pathStr);
if(size == SIZE_UNKNOWN) { //do roulette selection: pick dir with probability
//proportional to available size
@@ -355,7 +359,7 @@ public class LocalDirAllocator {
dir++;
}
dirNumLastAccessed = dir;
- returnPath = createPath(pathStr, checkWrite);
+ returnPath = createPath(path, checkWrite);
if (returnPath == null) {
totalAvailable -= availableOnDisk[dir];
availableOnDisk[dir] = 0; // skip this disk
@@ -366,7 +370,7 @@ public class LocalDirAllocator {
while (numDirsSearched < numDirs && returnPath == null) {
long capacity = dirDF[dirNumLastAccessed].getAvailable();
if (capacity > size) {
- returnPath = createPath(pathStr, checkWrite);
+ returnPath = createPath(path, checkWrite);
}
dirNumLastAccessed++;
dirNumLastAccessed = dirNumLastAccessed % numDirs;
@@ -409,15 +413,16 @@ public class LocalDirAllocator {
public synchronized Path getLocalPathToRead(String pathStr,
Configuration conf) throws IOException {
confChanged(conf);
- int numDirs = localDirs.length;
+ int numDirs = localDirsPath.length;
int numDirsSearched = 0;
//remove the leading slash from the path (to make sure that the uri
//resolution results in a valid path on the dir being checked)
if (pathStr.startsWith("/")) {
pathStr = pathStr.substring(1);
}
+ Path childPath = new Path(pathStr);
while (numDirsSearched < numDirs) {
- Path file = new Path(localDirs[numDirsSearched], pathStr);
+ Path file = new Path(localDirsPath[numDirsSearched], childPath);
if (localFS.exists(file)) {
return file;
}
@@ -434,10 +439,10 @@ public class LocalDirAllocator {
private final FileSystem fs;
private final String pathStr;
private int i = 0;
- private final String[] rootDirs;
+ private final Path[] rootDirs;
private Path next = null;
- private PathIterator(FileSystem fs, String pathStr, String[] rootDirs
+ private PathIterator(FileSystem fs, String pathStr, Path[] rootDirs
) throws IOException {
this.fs = fs;
this.pathStr = pathStr;
@@ -496,7 +501,7 @@ public class LocalDirAllocator {
if (pathStr.startsWith("/")) {
pathStr = pathStr.substring(1);
}
- return new PathIterator(localFS, pathStr, localDirs);
+ return new PathIterator(localFS, pathStr, localDirsPath);
}
/** We search through all the configured dirs for the file's existence
@@ -504,15 +509,16 @@ public class LocalDirAllocator {
*/
public synchronized boolean ifExists(String pathStr,Configuration conf) {
try {
- int numDirs = localDirs.length;
+ int numDirs = localDirsPath.length;
int numDirsSearched = 0;
//remove the leading slash from the path (to make sure that the uri
//resolution results in a valid path on the dir being checked)
if (pathStr.startsWith("/")) {
pathStr = pathStr.substring(1);
}
+ Path childPath = new Path(pathStr);
while (numDirsSearched < numDirs) {
- Path file = new Path(localDirs[numDirsSearched], pathStr);
+ Path file = new Path(localDirsPath[numDirsSearched], childPath);
if (localFS.exists(file)) {
return true;
}
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalFileSystem.java?rev=1079162&r1=1079161&r2=1079162&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalFileSystem.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/LocalFileSystem.java Tue Mar 8 04:41:03 2011
@@ -48,6 +48,18 @@ public class LocalFileSystem extends Che
super(rawLocalFileSystem);
rfs = rawLocalFileSystem;
}
+
+ /**
+ * Get file status.
+ */
+ public boolean exists(Path f) throws IOException {
+ File path = pathToFile(f);
+ if (path.exists()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
/** Convert a path to a File. */
public File pathToFile(Path path) {
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/Path.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/Path.java?rev=1079162&r1=1079161&r2=1079162&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/Path.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/Path.java Tue Mar 8 04:41:03 2011
@@ -157,8 +157,12 @@ public class Path implements Comparable
private String normalizePath(String path) {
// remove double slashes & backslashes
- path = path.replace("//", "/");
- path = path.replace("\\", "/");
+ if (path.indexOf("//") != -1) {
+ path = path.replace("//", "/");
+ }
+ if (path.indexOf("\\") != -1) {
+ path = path.replace("\\", "/");
+ }
// trim trailing slash from non-root path (ignoring windows drive)
int minLength = hasWindowsDrive(path, true) ? 4 : 1;