You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2010/05/18 01:51:56 UTC
svn commit: r945439 - in /hadoop/common/trunk: CHANGES.txt
src/java/core-default.xml
src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
src/java/org/apache/hadoop/fs/Trash.java
src/test/core/org/apache/hadoop/fs/TestTrash.java
Author: dhruba
Date: Mon May 17 23:51:56 2010
New Revision: 945439
URL: http://svn.apache.org/viewvc?rev=945439&view=rev
Log:
HADOOP-6761. The Trash Emptier has the ability to run more frequently.
(Dmytro Molkov via dhruba)
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/core-default.xml
hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/Trash.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=945439&r1=945438&r2=945439&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Mon May 17 23:51:56 2010
@@ -10,6 +10,9 @@ Trunk (unreleased changes)
HADOOP-6623. Add StringUtils.split for non-escaped single-character
separator. (Todd Lipcon via tomwhite)
+ HADOOP-6761. The Trash Emptier has the ability to run more frequently.
+ (Dmytro Molkov via dhruba)
+
BUG FIXES
HADOOP-6612. Protocols RefreshUserToGroupMappingsProtocol and
Modified: hadoop/common/trunk/src/java/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/core-default.xml?rev=945439&r1=945438&r2=945439&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/core-default.xml (original)
+++ hadoop/common/trunk/src/java/core-default.xml Mon May 17 23:51:56 2010
@@ -162,12 +162,24 @@
<property>
<name>fs.trash.interval</name>
<value>0</value>
- <description>Number of minutes between trash checkpoints.
+ <description>Number of minutes after which the checkpoint
+ gets deleted.
If zero, the trash feature is disabled.
</description>
</property>
<property>
+ <name>fs.trash.checkpoint.interval</name>
+ <value>0</value>
+ <description>Number of minutes between trash checkpoints.
+ Should be smaller or equal to fs.trash.interval.
+ Every time the checkpointer runs it creates a new checkpoint
+ out of current and removes checkpoints created more than
+ fs.trash.interval minutes ago.
+ </description>
+</property>
+
+<property>
<name>fs.file.impl</name>
<value>org.apache.hadoop.fs.LocalFileSystem</value>
<description>The FileSystem for file: uris.</description>
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=945439&r1=945438&r2=945439&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Mon May 17 23:51:56 2010
@@ -55,6 +55,8 @@ public class CommonConfigurationKeys {
public static final String FS_CLIENT_BUFFER_DIR_KEY =
"fs.client.buffer.dir";
+ public static final String FS_TRASH_CHECKPOINT_INTERVAL_KEY = "fs.trash.checkpoint.interval";
+ public static final long FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT = 0;
//TBD: Code is not updated to use following keys.
//These keys will be used in later versions
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/Trash.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/Trash.java?rev=945439&r1=945438&r2=945439&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/Trash.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/Trash.java Mon May 17 23:51:56 2010
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.StringUtils;
@@ -47,18 +48,19 @@ public class Trash extends Configured {
private static final Path CURRENT = new Path("Current");
private static final Path TRASH = new Path(".Trash/");
- private static final Path HOMES = new Path("/user/");
+
private static final FsPermission PERMISSION =
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE);
- private static final DateFormat CHECKPOINT = new SimpleDateFormat("yyMMddHHmm");
+ private static final DateFormat CHECKPOINT = new SimpleDateFormat("yyMMddHHmmss");
private static final int MSECS_PER_MINUTE = 60*1000;
private final FileSystem fs;
private final Path trash;
private final Path current;
- private final long interval;
+ private final long deletionInterval;
+ private final Path homesParent;
/** Construct a trash can accessor.
* @param conf a Configuration
@@ -74,16 +76,22 @@ public class Trash extends Configured {
super(conf);
this.fs = fs;
this.trash = new Path(fs.getHomeDirectory(), TRASH);
+ this.homesParent = fs.getHomeDirectory().getParent();
this.current = new Path(trash, CURRENT);
- this.interval = conf.getLong("fs.trash.interval", 60) * MSECS_PER_MINUTE;
+ this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY,
+ FS_TRASH_INTERVAL_DEFAULT) *
+ MSECS_PER_MINUTE);
}
private Trash(Path home, Configuration conf) throws IOException {
super(conf);
this.fs = home.getFileSystem(conf);
this.trash = new Path(home, TRASH);
+ this.homesParent = home.getParent();
this.current = new Path(trash, CURRENT);
- this.interval = conf.getLong("fs.trash.interval", 60) * MSECS_PER_MINUTE;
+ this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY,
+ FS_TRASH_INTERVAL_DEFAULT) *
+ MSECS_PER_MINUTE);
}
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
@@ -94,7 +102,7 @@ public class Trash extends Configured {
* @return false if the item is already in the trash or trash is disabled
*/
public boolean moveToTrash(Path path) throws IOException {
- if (interval == 0)
+ if (deletionInterval == 0)
return false;
if (!path.isAbsolute()) // make path absolute
@@ -197,7 +205,7 @@ public class Trash extends Configured {
continue;
}
- if ((now - interval) > time) {
+ if ((now - deletionInterval) > time) {
if (fs.delete(path, true)) {
LOG.info("Deleted trash checkpoint: "+dir);
} else {
@@ -225,34 +233,43 @@ public class Trash extends Configured {
private class Emptier implements Runnable {
private Configuration conf;
- private long interval;
+ private long emptierInterval;
Emptier(Configuration conf) throws IOException {
this.conf = conf;
- this.interval = conf.getLong("fs.trash.interval", 0) * MSECS_PER_MINUTE;
+ this.emptierInterval = (long) (conf.getFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY,
+ FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) *
+ MSECS_PER_MINUTE);
+ if (this.emptierInterval > deletionInterval ||
+ this.emptierInterval == 0) {
+ LOG.warn("The configured interval for checkpoint is " +
+ this.emptierInterval + " minutes." +
+ " Using interval of " + deletionInterval +
+ " minutes that is used for deletion instead");
+ this.emptierInterval = deletionInterval;
+ }
}
public void run() {
- if (interval == 0)
+ if (emptierInterval == 0)
return; // trash disabled
-
long now = System.currentTimeMillis();
long end;
while (true) {
- end = ceiling(now, interval);
+ end = ceiling(now, emptierInterval);
try { // sleep for interval
Thread.sleep(end - now);
} catch (InterruptedException e) {
break; // exit on interrupt
}
-
+
try {
now = System.currentTimeMillis();
if (now >= end) {
FileStatus[] homes = null;
try {
- homes = fs.listStatus(HOMES); // list all home dirs
+ homes = fs.listStatus(homesParent); // list all home dirs
} catch (IOException e) {
LOG.warn("Trash can't list homes: "+e+" Sleeping.");
continue;
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java?rev=945439&r1=945438&r2=945439&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/TestTrash.java Mon May 17 23:51:56 2010
@@ -18,11 +18,15 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
+
import java.io.DataOutputStream;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.net.URI;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
import junit.framework.TestCase;
@@ -89,7 +93,7 @@ public class TestTrash extends TestCase
protected static void trashShell(final FileSystem fs, final Path base)
throws IOException {
Configuration conf = new Configuration();
- conf.set("fs.trash.interval", "10"); // 10 minute
+ conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute
conf.set("fs.default.name", fs.getUri().toString());
FsShell shell = new FsShell();
shell.setConf(conf);
@@ -376,7 +380,7 @@ public class TestTrash extends TestCase
}
public static void trashNonDefaultFS(Configuration conf) throws IOException {
- conf.set("fs.trash.interval", "10"); // 10 minute
+ conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute
// attempt non-default FileSystem trash
{
final FileSystem lfs = FileSystem.getLocal(conf);
@@ -414,6 +418,67 @@ public class TestTrash extends TestCase
trashNonDefaultFS(conf);
}
+ public void testTrashEmptier() throws Exception {
+ Configuration conf = new Configuration();
+ // Trash with 12 second deletes and 6 seconds checkpoints
+ conf.set(FS_TRASH_INTERVAL_KEY, "0.2"); // 12 seconds
+ conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+ conf.set(FS_TRASH_CHECKPOINT_INTERVAL_KEY, "0.1"); // 6 seconds
+ Trash trash = new Trash(conf);
+
+ // Start Emptier in background
+ Runnable emptier = trash.getEmptier();
+ Thread emptierThread = new Thread(emptier);
+ emptierThread.start();
+
+ FileSystem fs = FileSystem.getLocal(conf);
+ conf.set("fs.default.name", fs.getUri().toString());
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ shell.init();
+ // First create a new directory with mkdirs
+ Path myPath = new Path(TEST_DIR, "test/mkdirs");
+ mkdir(fs, myPath);
+ int fileIndex = 0;
+ Set<String> checkpoints = new HashSet<String>();
+ while (true) {
+ // Create a file with a new name
+ Path myFile = new Path(TEST_DIR, "test/mkdirs/myFile" + fileIndex++);
+ writeFile(fs, myFile);
+
+ // Delete the file to trash
+ String[] args = new String[2];
+ args[0] = "-rm";
+ args[1] = myFile.toString();
+ int val = -1;
+ try {
+ val = shell.run(args);
+ } catch (Exception e) {
+ System.err.println("Exception raised from Trash.run " +
+ e.getLocalizedMessage());
+ }
+ assertTrue(val == 0);
+
+ Path trashDir = shell.getCurrentTrashDir();
+ FileStatus files[] = fs.listStatus(trashDir.getParent());
+ // Scan files in .Trash and add them to set of checkpoints
+ for (FileStatus file : files) {
+ String fileName = file.getPath().getName();
+ checkpoints.add(fileName);
+ }
+ // If checkpoints has 4 objects it is Current + 3 checkpoint directories
+ if (checkpoints.size() == 4) {
+ // The actual contents should be smaller since the last checkpoint
+ // should've been deleted and Current might not have been recreated yet
+ assertTrue(checkpoints.size() > files.length);
+ break;
+ }
+ Thread.sleep(5000);
+ }
+ emptierThread.interrupt();
+ emptierThread.join();
+ }
+
/**
* @see TestCase#tearDown()
*/
@@ -428,7 +493,7 @@ public class TestTrash extends TestCase
static class TestLFS extends LocalFileSystem {
Path home;
TestLFS() {
- this(TEST_DIR);
+ this(new Path(TEST_DIR, "user/test"));
}
TestLFS(Path home) {
super();
@@ -451,7 +516,7 @@ public class TestTrash extends TestCase
FileSystem fs = FileSystem.getLocal(conf);
conf.set("fs.default.name", fs.getUri().toString());
- conf.set("fs.trash.interval", "10"); //minutes..
+ conf.set(FS_TRASH_INTERVAL_KEY, "10"); //minutes..
FsShell shell = new FsShell();
shell.setConf(conf);
//Path trashRoot = null;