You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2008/05/22 07:03:11 UTC
svn commit: r659005 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
src/java/org/apache/hadoop/fs/FileSystem.java
src/test/org/apache/hadoop/dfs/TestFileCreation.java
Author: dhruba
Date: Wed May 21 22:03:10 2008
New Revision: 659005
URL: http://svn.apache.org/viewvc?rev=659005&view=rev
Log:
HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
handling of temporary files in HDFS. (dhruba)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed May 21 22:03:10 2008
@@ -94,6 +94,9 @@
HADOOP-3336. Direct a subset of annotated FSNamesystem calls for audit
logging. (cdouglas)
+ HADOOP-3400. A new API FileSystem.deleteOnExit() that facilitates
+ handling of temporary files in HDFS. (dhruba)
+
IMPROVEMENTS
HADOOP-2928. Remove deprecated FileSystem.getContentLength().
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DistributedFileSystem.java Wed May 21 22:03:10 2008
@@ -194,6 +194,7 @@
/** {@inheritDoc} */
public void close() throws IOException {
try {
+ super.processDeleteOnExit();
dfs.close();
} finally {
super.close();
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed May 21 22:03:10 2008
@@ -69,6 +69,12 @@
* The statistics for this file system.
*/
protected final Statistics statistics;
+
+ /**
+ * A cache of files that should be deleted when filsystem is closed
+ * or the JVM is exited.
+ */
+ private Set<Path> deleteOnExit = new TreeSet<Path>();
/**
* Parse the cmd-line args, starting at i. Remove consumed args
@@ -591,6 +597,41 @@
* @throws IOException
*/
public abstract boolean delete(Path f, boolean recursive) throws IOException;
+
+ /** Mark a path to be deleted when FileSystem is closed or JVM exits.
+ * The path has to exist in the file system.
+ *
+ * @param f the path to delete.
+ * @return true if deleteOnExit is successful, otherwise false.
+ * @throws IOException
+ */
+ public boolean deleteOnExit(Path f) throws IOException {
+ if (!exists(f)) {
+ return false;
+ }
+ synchronized (deleteOnExit) {
+ deleteOnExit.add(f);
+ }
+ return true;
+ }
+
+ // Delete all files that were marked as delete-on-exit. This recursively
+ // deletes all files in the specified paths.
+ //
+ protected void processDeleteOnExit() {
+ synchronized (deleteOnExit) {
+ for (Iterator<Path> iter = deleteOnExit.iterator(); iter.hasNext();) {
+ Path path = iter.next();
+ try {
+ delete(path, true);
+ }
+ catch (IOException e) {
+ LOG.info("Ignoring failure to deleteOnExit for path " + path);
+ }
+ iter.remove();
+ }
+ }
+ }
/** Check if exists.
* @param f source file
@@ -1172,6 +1213,8 @@
* release any held locks.
*/
public void close() throws IOException {
+ // delete all files that were marked as delete-on-exit.
+ processDeleteOnExit();
CACHE.remove(new Cache.Key(this), this);
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java?rev=659005&r1=659004&r2=659005&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFileCreation.java Wed May 21 22:03:10 2008
@@ -203,6 +203,71 @@
}
/**
+ * Test deleteOnExit
+ */
+ public void testDeleteOnExit() throws IOException {
+ Configuration conf = new Configuration();
+ if (simulatedStorage) {
+ conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
+ }
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 1, true, null);
+ FileSystem fs = cluster.getFileSystem();
+ FileSystem localfs = FileSystem.getLocal(conf);
+
+ try {
+
+ // Creates files in HDFS and local file system.
+ //
+ Path file1 = new Path("filestatus.dat");
+ Path file2 = new Path("filestatus2.dat");
+ Path file3 = new Path("filestatus3.dat");
+ FSDataOutputStream stm1 = createFile(fs, file1, 1);
+ FSDataOutputStream stm2 = createFile(fs, file2, 1);
+ FSDataOutputStream stm3 = createFile(localfs, file3, 1);
+ System.out.println("DeleteOnExit: Created files.");
+
+ // write to files and close. Purposely, do not close file2.
+ writeFile(stm1);
+ writeFile(stm3);
+ stm1.close();
+ stm3.close();
+
+ // set delete on exit flag on files.
+ fs.deleteOnExit(file1);
+ fs.deleteOnExit(file2);
+ localfs.deleteOnExit(file3);
+
+ // close the file system. This should make the above files
+ // disappear.
+ fs.close();
+ localfs.close();
+ fs = null;
+ localfs = null;
+
+ // reopen file system and verify that file does not exist.
+ fs = cluster.getFileSystem();
+ localfs = FileSystem.getLocal(conf);
+
+ assertTrue(file1 + " still exists inspite of deletOnExit set.",
+ !fs.exists(file1));
+ assertTrue(file2 + " still exists inspite of deletOnExit set.",
+ !fs.exists(file2));
+ assertTrue(file3 + " still exists inspite of deletOnExit set.",
+ !localfs.exists(file3));
+ System.out.println("DeleteOnExit successful.");
+
+ } finally {
+ if (fs != null) {
+ fs.close();
+ }
+ if (localfs != null) {
+ localfs.close();
+ }
+ cluster.shutdown();
+ }
+ }
+
+ /**
* Test that file data does not become corrupted even in the face of errors.
*/
public void testFileCreationError1() throws IOException {