You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2012/08/18 01:22:17 UTC
svn commit: r1374472 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/fs/ftp/
src/main/java/org/apache/hadoop/fs/local/ src/main/resources/
src/test/java/org/apache/ha...
Author: eli
Date: Fri Aug 17 23:22:17 2012
New Revision: 1374472
URL: http://svn.apache.org/viewvc?rev=1374472&view=rev
Log:
HADOOP-8689. Make trash a server side configuration option. Contributed by Eli Collins
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 23:22:17 2012
@@ -198,6 +198,8 @@ Branch-2 ( Unreleased changes )
HADOOP-8388. Remove unused BlockLocation serialization.
(Colin Patrick McCabe via eli)
+ HADOOP-8689. Make trash a server side configuration option. (eli)
+
NEW FEATURES
HDFS-3042. Automatic failover support for NameNode HA (todd)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java Fri Aug 17 23:22:17 2012
@@ -661,7 +661,9 @@ public abstract class FileSystem extends
64 * 1024,
getDefaultReplication(),
conf.getInt("io.file.buffer.size", 4096),
- false);
+ false,
+ // NB: ignoring the client trash configuration
+ CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT);
}
/**
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java Fri Aug 17 23:22:17 2012
@@ -49,19 +49,21 @@ public class FsServerDefaults implements
private short replication;
private int fileBufferSize;
private boolean encryptDataTransfer;
+ private long trashInterval;
public FsServerDefaults() {
}
public FsServerDefaults(long blockSize, int bytesPerChecksum,
int writePacketSize, short replication, int fileBufferSize,
- boolean encryptDataTransfer) {
+ boolean encryptDataTransfer, long trashInterval) {
this.blockSize = blockSize;
this.bytesPerChecksum = bytesPerChecksum;
this.writePacketSize = writePacketSize;
this.replication = replication;
this.fileBufferSize = fileBufferSize;
this.encryptDataTransfer = encryptDataTransfer;
+ this.trashInterval = trashInterval;
}
public long getBlockSize() {
@@ -88,6 +90,10 @@ public class FsServerDefaults implements
return encryptDataTransfer;
}
+ public long getTrashInterval() {
+ return trashInterval;
+ }
+
// /////////////////////////////////////////
// Writable
// /////////////////////////////////////////
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java Fri Aug 17 23:22:17 2012
@@ -34,7 +34,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -66,6 +65,7 @@ public class TrashPolicyDefault extends
private Path current;
private Path homesParent;
+ private long emptierInterval;
public TrashPolicyDefault() { }
@@ -79,8 +79,27 @@ public class TrashPolicyDefault extends
this.trash = new Path(home, TRASH);
this.homesParent = home.getParent();
this.current = new Path(trash, CURRENT);
- this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY,
- FS_TRASH_INTERVAL_DEFAULT) * MSECS_PER_MINUTE);
+ long trashInterval = 0;
+ try {
+ trashInterval = fs.getServerDefaults(home).getTrashInterval();
+ } catch (IOException ioe) {
+ LOG.warn("Unable to get server defaults", ioe);
+ }
+ // If the trash interval is not configured or is disabled on the
+ // server side then check the config which may be client side.
+ if (0 == trashInterval) {
+ this.deletionInterval = (long)(conf.getFloat(
+ FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)
+ * MSECS_PER_MINUTE);
+ } else {
+ this.deletionInterval = trashInterval * MSECS_PER_MINUTE;
+ }
+ // For the checkpoint interval use the given config instead of
+ // checking the server as it's OK if a client starts an emptier
+ // with a different interval than the server.
+ this.emptierInterval = (long)(conf.getFloat(
+ FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
+ * MSECS_PER_MINUTE);
}
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
@@ -89,7 +108,7 @@ public class TrashPolicyDefault extends
@Override
public boolean isEnabled() {
- return (deletionInterval != 0);
+ return deletionInterval != 0;
}
@Override
@@ -223,7 +242,7 @@ public class TrashPolicyDefault extends
@Override
public Runnable getEmptier() throws IOException {
- return new Emptier(getConf());
+ return new Emptier(getConf(), emptierInterval);
}
private class Emptier implements Runnable {
@@ -231,16 +250,14 @@ public class TrashPolicyDefault extends
private Configuration conf;
private long emptierInterval;
- Emptier(Configuration conf) throws IOException {
+ Emptier(Configuration conf, long emptierInterval) throws IOException {
this.conf = conf;
- this.emptierInterval = (long) (conf.getFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY,
- FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) *
- MSECS_PER_MINUTE);
- if (this.emptierInterval > deletionInterval ||
- this.emptierInterval == 0) {
- LOG.warn("The configured interval for checkpoint is " +
- this.emptierInterval + " minutes." +
- " Using interval of " + deletionInterval +
+ this.emptierInterval = emptierInterval;
+ if (emptierInterval > deletionInterval || emptierInterval == 0) {
+ LOG.info("The configured checkpoint interval is " +
+ (emptierInterval / MSECS_PER_MINUTE) + " minutes." +
+ " Using an interval of " +
+ (deletionInterval / MSECS_PER_MINUTE) +
" minutes that is used for deletion instead");
this.emptierInterval = deletionInterval;
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java Fri Aug 17 23:22:17 2012
@@ -45,6 +45,7 @@ public class FtpConfigKeys extends Commo
"ftp.client-write-packet-size";
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
+ public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
protected static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
@@ -53,7 +54,8 @@ public class FtpConfigKeys extends Commo
CLIENT_WRITE_PACKET_SIZE_DEFAULT,
REPLICATION_DEFAULT,
STREAM_BUFFER_SIZE_DEFAULT,
- ENCRYPT_DATA_TRANSFER_DEFAULT);
+ ENCRYPT_DATA_TRANSFER_DEFAULT,
+ FS_TRASH_INTERVAL_DEFAULT);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java Fri Aug 17 23:22:17 2012
@@ -44,6 +44,7 @@ public class LocalConfigKeys extends Com
"file.client-write-packet-size";
public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024;
public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false;
+ public static final long FS_TRASH_INTERVAL_DEFAULT = 0;
public static FsServerDefaults getServerDefaults() throws IOException {
return new FsServerDefaults(
@@ -52,7 +53,8 @@ public class LocalConfigKeys extends Com
CLIENT_WRITE_PACKET_SIZE_DEFAULT,
REPLICATION_DEFAULT,
STREAM_BUFFER_SIZE_DEFAULT,
- ENCRYPT_DATA_TRANSFER_DEFAULT);
+ ENCRYPT_DATA_TRANSFER_DEFAULT,
+ FS_TRASH_INTERVAL_DEFAULT);
}
}
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml Fri Aug 17 23:22:17 2012
@@ -351,8 +351,12 @@
<name>fs.trash.interval</name>
<value>0</value>
<description>Number of minutes after which the checkpoint
- gets deleted.
- If zero, the trash feature is disabled.
+ gets deleted. If zero, the trash feature is disabled.
+ This option may be configured both on the server and the
+ client. If trash is disabled server side then the client
+ side configuration is checked. If trash is enabled on the
+ server side then the value configured on the server is
+ used and the client configuration value is ignored.
</description>
</property>
@@ -360,7 +364,8 @@
<name>fs.trash.checkpoint.interval</name>
<value>0</value>
<description>Number of minutes between trash checkpoints.
- Should be smaller or equal to fs.trash.interval.
+ Should be smaller or equal to fs.trash.interval. If zero,
+ the value is set to the value of fs.trash.interval.
Every time the checkpointer runs it creates a new checkpoint
out of current and removes checkpoints created more than
fs.trash.interval minutes ago.
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java?rev=1374472&r1=1374471&r2=1374472&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java Fri Aug 17 23:22:17 2012
@@ -111,10 +111,10 @@ public class TestTrash extends TestCase
throws IOException {
FileSystem fs = FileSystem.get(conf);
- conf.set(FS_TRASH_INTERVAL_KEY, "0"); // disabled
+ conf.setLong(FS_TRASH_INTERVAL_KEY, 0); // disabled
assertFalse(new Trash(conf).isEnabled());
- conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute
+ conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute
assertTrue(new Trash(conf).isEnabled());
FsShell shell = new FsShell();
@@ -435,7 +435,7 @@ public class TestTrash extends TestCase
}
public static void trashNonDefaultFS(Configuration conf) throws IOException {
- conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute
+ conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute
// attempt non-default FileSystem trash
{
final FileSystem lfs = FileSystem.getLocal(conf);
@@ -580,7 +580,7 @@ public class TestTrash extends TestCase
FileSystem fs = FileSystem.getLocal(conf);
conf.set("fs.defaultFS", fs.getUri().toString());
- conf.set(FS_TRASH_INTERVAL_KEY, "10"); //minutes..
+ conf.setLong(FS_TRASH_INTERVAL_KEY, 10); //minutes..
FsShell shell = new FsShell();
shell.setConf(conf);
//Path trashRoot = null;