You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/06/18 23:45:10 UTC
svn commit: r548494 - in /lucene/hadoop/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/DFSClient.java
src/java/org/apache/hadoop/fs/LocalDirAllocator.java
Author: cutting
Date: Mon Jun 18 14:45:08 2007
New Revision: 548494
URL: http://svn.apache.org/viewvc?view=rev&rev=548494
Log:
HADOOP-1372. Use LocalDirAllocator for HDFS temporary block files. Contributed by Dhruba.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 18 14:45:08 2007
@@ -160,6 +160,10 @@
property in "mapred.child.java.opts" configuration property.
(Enis Soztutar via cutting)
+ 51. HADOOP-1372. Use LocalDirAllocator for HDFS temporary block
+ files, so that disk space, writability, etc. is considered.
+ (Dhruba Borthakur via cutting)
+
Release 0.13.0 - 2007-06-08
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Mon Jun 18 14:45:08 2007
@@ -58,6 +58,7 @@
private Configuration conf;
private long defaultBlockSize;
private short defaultReplication;
+ private LocalDirAllocator dirAllocator;
/**
* A map from name -> DFSOutputStream of files that are currently being
@@ -159,6 +160,7 @@
}
defaultBlockSize = conf.getLong("dfs.block.size", DEFAULT_BLOCK_SIZE);
defaultReplication = (short) conf.getInt("dfs.replication", 3);
+ dirAllocator = new LocalDirAllocator("dfs.client.buffer.dir");
this.leaseChecker = new Daemon(new LeaseChecker());
this.leaseChecker.start();
}
@@ -1167,12 +1169,11 @@
}
private File newBackupFile() throws IOException {
- File file = conf.getFile("dfs.client.buffer.dir", "tmp");
- File dir = file.getParentFile();
- String prefix = "client-" + Math.abs(r.nextLong());
- String suffix = ".tmp";
- File result = File.createTempFile(prefix, suffix, dir);
- result.deleteOnExit();
+ String name = "tmp" + File.separator +
+ "client-" + Math.abs(r.nextLong());
+ File result = dirAllocator.createTmpFileForWrite(name,
+ 2 * blockSize,
+ conf);
return result;
}
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java?view=diff&rev=548494&r1=548493&r2=548494
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/LocalDirAllocator.java Mon Jun 18 14:45:08 2007
@@ -137,6 +137,23 @@
AllocatorPerContext context = obtainContext(contextCfgItemName);
return context.getLocalPathToRead(pathStr, conf);
}
+
+ /** Creates a temporary file in the local FS. Pass size as -1 if not known
+ * apriori. We round-robin over the set of disks (via the configured dirs)
+ * and select the first complete path which has enough space. A file is
+ * created on this directory. The file is guaranteed to go away when the
+ * JVM exits.
+ * @param pathStr prefix for the temporary file
+ * @param size the size of the file that is going to be written
+ * @param conf the Configuration object
+ * @return a unique temporary file
+ * @throws IOException
+ */
+ public File createTmpFileForWrite(String pathStr, long size,
+ Configuration conf) throws IOException {
+ AllocatorPerContext context = obtainContext(contextCfgItemName);
+ return context.createTmpFileForWrite(pathStr, size, conf);
+ }
/** Method to check whether a context is valid
* @param contextCfgItemName
@@ -245,6 +262,25 @@
//no path found
throw new DiskErrorException("Could not find any valid local " +
"directory for " + pathStr);
+ }
+
+ /** Creates a file on the local FS. Pass size as -1 if not known apriori. We
+ * round-robin over the set of disks (via the configured dirs) and return
+ * a file on the first path which has enough space. The file is guaranteed
+ * to go away when the JVM exits.
+ */
+ public File createTmpFileForWrite(String pathStr, long size,
+ Configuration conf) throws IOException {
+
+ // find an appropriate directory
+ Path path = getLocalPathForWrite(pathStr, size, conf);
+ File dir = new File(path.getParent().toUri().getPath());
+ String prefix = path.getName();
+
+ // create a temp file on this directory
+ File result = File.createTempFile(prefix, null, dir);
+ result.deleteOnExit();
+ return result;
}
/** Get a path from the local FS for reading. We search through all the