You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2016/10/14 06:05:36 UTC
hadoop git commit: HADOOP-13686. Adding additional unit test for
Trash (I). Contributed by Weiwei Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 5a5a72473 -> dbe663d52
HADOOP-13686. Adding additional unit test for Trash (I). Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dbe663d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dbe663d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dbe663d5
Branch: refs/heads/trunk
Commit: dbe663d5241feea0c88a3a9391ad48a029001d94
Parents: 5a5a724
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 13 23:05:16 2016 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Oct 13 23:05:16 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/fs/TrashPolicyDefault.java | 11 +-
.../java/org/apache/hadoop/fs/TestTrash.java | 352 ++++++++++++++++++-
2 files changed, 356 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dbe663d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 72222be..4f4c937 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Time;
+import com.google.common.annotations.VisibleForTesting;
+
/** Provides a <i>trash</i> feature. Files are moved to a user's trash
* directory, a subdirectory of their home directory named ".Trash". Files are
* initially moved to a <i>current</i> sub-directory of the trash directory.
@@ -215,7 +217,7 @@ public class TrashPolicyDefault extends TrashPolicy {
return new Emptier(getConf(), emptierInterval);
}
- private class Emptier implements Runnable {
+ protected class Emptier implements Runnable {
private Configuration conf;
private long emptierInterval;
@@ -223,7 +225,7 @@ public class TrashPolicyDefault extends TrashPolicy {
Emptier(Configuration conf, long emptierInterval) throws IOException {
this.conf = conf;
this.emptierInterval = emptierInterval;
- if (emptierInterval > deletionInterval || emptierInterval == 0) {
+ if (emptierInterval > deletionInterval || emptierInterval <= 0) {
LOG.info("The configured checkpoint interval is " +
(emptierInterval / MSECS_PER_MINUTE) + " minutes." +
" Using an interval of " +
@@ -287,6 +289,11 @@ public class TrashPolicyDefault extends TrashPolicy {
private long floor(long time, long interval) {
return (time / interval) * interval;
}
+
+ @VisibleForTesting
+ protected long getEmptierInterval() {
+ return this.emptierInterval/MSECS_PER_MINUTE;
+ }
}
private void createCheckpoint(Path trashRoot, Date date) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dbe663d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 338aff6..7a5b25e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -29,13 +29,19 @@ import java.net.URI;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.HashSet;
+import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.TrashPolicyDefault.Emptier;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
+import org.junit.Before;
+import org.junit.Test;
/**
* This class tests commands from Trash.
@@ -45,6 +51,13 @@ public class TestTrash extends TestCase {
private final static Path TEST_DIR = new Path(GenericTestUtils.getTempPath(
"testTrash"));
+ @Before
+ public void setUp() throws IOException {
+ // ensure each test initiates a FileSystem instance,
+ // avoid getting an old instance from cache.
+ FileSystem.closeAll();
+ }
+
protected static Path mkdir(FileSystem fs, Path p) throws IOException {
assertTrue(fs.mkdirs(p));
assertTrue(fs.exists(p));
@@ -516,6 +529,81 @@ public class TestTrash extends TestCase {
assertTrue(trash.getTrashPolicy().getClass().equals(TestTrashPolicy.class));
}
+ @Test
+ public void testCheckpointInterval() throws IOException {
+ // Verify if fs.trash.checkpoint.interval is set to positive number
+ // but bigger than fs.trash.interval,
+ // the value should be reset to fs.trash.interval
+ verifyDefaultPolicyIntervalValues(10, 12, 10);
+
+ // Verify if fs.trash.checkpoint.interval is set to positive number
+ // and smaller than fs.trash.interval, the value should be respected
+ verifyDefaultPolicyIntervalValues(10, 5, 5);
+
+ // Verify if fs.trash.checkpoint.interval sets to 0
+ // the value should be reset to fs.trash.interval
+ verifyDefaultPolicyIntervalValues(10, 0, 10);
+
+ // Verify if fs.trash.checkpoint.interval sets to a negative number
+ // the value should be reset to fs.trash.interval
+ verifyDefaultPolicyIntervalValues(10, -1, 10);
+ }
+
+ @Test
+ public void testMoveEmptyDirToTrash() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass(FS_FILE_IMPL_KEY,
+ RawLocalFileSystem.class,
+ FileSystem.class);
+ conf.setLong(FS_TRASH_INTERVAL_KEY, 1); // 1 min
+ FileSystem fs = FileSystem.get(conf);
+ verifyMoveEmptyDirToTrash(fs, conf);
+ }
+
+ /**
+ * Simulate the carrier process of the trash emptier restarts,
+ * verify it honors the <b>fs.trash.interval</b> before and after restart.
+ * @throws Exception
+ */
+ @Test
+ public void testTrashRestarts() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass("fs.trash.classname",
+ AuditableTrashPolicy.class,
+ TrashPolicy.class);
+ conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+ conf.set(FS_TRASH_INTERVAL_KEY, "50"); // in milliseconds for test
+ Trash trash = new Trash(conf);
+ // create 5 checkpoints
+ for(int i=0; i<5; i++) {
+ trash.checkpoint();
+ }
+
+ // Run the trash emptier for 120ms, it should run
+ // 2 times deletion as the interval is 50ms.
+ // Verify the checkpoints number when shutting down the emptier.
+ verifyAuditableTrashEmptier(trash, 120, 3);
+
+ // reconfigure the interval to 100 ms
+ conf.set(FS_TRASH_INTERVAL_KEY, "100");
+ Trash trashNew = new Trash(conf);
+
+ // Run the trash emptier for 120ms, it should run
+ // 1 time deletion.
+ verifyAuditableTrashEmptier(trashNew, 120, 2);
+ }
+
+ @Test
+ public void testTrashPermission() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setClass("fs.trash.classname",
+ TrashPolicyDefault.class,
+ TrashPolicy.class);
+ conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+ conf.set(FS_TRASH_INTERVAL_KEY, "0.2");
+ verifyTrashPermission(FileSystem.getLocal(conf), conf);
+ }
+
public void testTrashEmptier() throws Exception {
Configuration conf = new Configuration();
// Trash with 12 second deletes and 6 seconds checkpoints
@@ -679,12 +767,143 @@ public class TestTrash extends TestCase {
long factoredTime = first*factor;
assertTrue(iterTime<factoredTime); //no more then twice of median first 10
}
- }
+ }
}
-
- public static void main(String [] arg) throws IOException{
- // run performance piece as a separate test
- performanceTestDeleteSameFile();
+
+ public static void verifyMoveEmptyDirToTrash(FileSystem fs,
+ Configuration conf) throws IOException {
+ Path caseRoot = new Path(
+ GenericTestUtils.getTempPath("testUserTrash"));
+ Path testRoot = new Path(caseRoot, "trash-users");
+ Path emptyDir = new Path(testRoot, "empty-dir");
+ try (FileSystem fileSystem = fs){
+ fileSystem.mkdirs(emptyDir);
+ Trash trash = new Trash(fileSystem, conf);
+ // Make sure trash root is clean
+ Path trashRoot = trash.getCurrentTrashDir(emptyDir);
+ fileSystem.delete(trashRoot, true);
+ // Move to trash should be succeed
+ assertTrue("Move an empty directory to trash failed",
+ trash.moveToTrash(emptyDir));
+ // Verify the empty dir is removed
+ assertFalse("The empty directory still exists on file system",
+ fileSystem.exists(emptyDir));
+ emptyDir = fileSystem.makeQualified(emptyDir);
+ Path dirInTrash = Path.mergePaths(trashRoot, emptyDir);
+ assertTrue("Directory wasn't moved to trash",
+ fileSystem.exists(dirInTrash));
+ FileStatus[] flist = fileSystem.listStatus(dirInTrash);
+ assertTrue("Directory is not empty",
+ flist!= null && flist.length == 0);
+ }
+ }
+
+ /**
+ * Create a bunch of files and set with different permission, after
+ * moved to trash, verify the location in trash directory is expected
+ * and the permission is reserved.
+ *
+ * @throws IOException
+ */
+ public static void verifyTrashPermission(FileSystem fs, Configuration conf)
+ throws IOException {
+ Path caseRoot = new Path(
+ GenericTestUtils.getTempPath("testTrashPermission"));
+ try (FileSystem fileSystem = fs){
+ Trash trash = new Trash(fileSystem, conf);
+ FileSystemTestWrapper wrapper =
+ new FileSystemTestWrapper(fileSystem);
+
+ short[] filePermssions = {
+ (short) 0600,
+ (short) 0644,
+ (short) 0660,
+ (short) 0700,
+ (short) 0750,
+ (short) 0755,
+ (short) 0775,
+ (short) 0777
+ };
+
+ for(int i=0; i<filePermssions.length; i++) {
+ // Set different permission to files
+ FsPermission fsPermission = new FsPermission(filePermssions[i]);
+ Path file = new Path(caseRoot, "file" + i);
+ byte[] randomBytes = new byte[new Random().nextInt(10)];
+ wrapper.writeFile(file, randomBytes);
+ wrapper.setPermission(file, fsPermission);
+
+ // Move file to trash
+ trash.moveToTrash(file);
+
+ // Verify the file is moved to trash, at expected location
+ Path trashDir = trash.getCurrentTrashDir(file);
+ if(!file.isAbsolute()) {
+ file = wrapper.makeQualified(file);
+ }
+ Path fileInTrash = Path.mergePaths(trashDir, file);
+ FileStatus fstat = wrapper.getFileStatus(fileInTrash);
+ assertTrue(String.format("File %s is not moved to trash",
+ fileInTrash.toString()),
+ wrapper.exists(fileInTrash));
+ // Verify permission not change
+ assertTrue(String.format("Expected file: %s is %s, but actual is %s",
+ fileInTrash.toString(),
+ fsPermission.toString(),
+ fstat.getPermission().toString()),
+ fstat.getPermission().equals(fsPermission));
+ }
+
+ // Verify the trash directory can be removed
+ Path trashRoot = trash.getCurrentTrashDir();
+ assertTrue(wrapper.delete(trashRoot, true));
+ }
+ }
+
+ private void verifyDefaultPolicyIntervalValues(long trashInterval,
+ long checkpointInterval, long expectedInterval) throws IOException {
+ Configuration conf = new Configuration();
+ conf.setLong(FS_TRASH_INTERVAL_KEY, trashInterval);
+ conf.set("fs.trash.classname", TrashPolicyDefault.class.getName());
+ conf.setLong(FS_TRASH_CHECKPOINT_INTERVAL_KEY, checkpointInterval);
+ Trash trash = new Trash(conf);
+ Emptier emptier = (Emptier)trash.getEmptier();
+ assertEquals(expectedInterval, emptier.getEmptierInterval());
+ }
+
+ /**
+ * Launch the {@link Trash} emptier for given milliseconds,
+ * verify the number of checkpoints is expected.
+ */
+ private void verifyAuditableTrashEmptier(Trash trash,
+ long timeAlive,
+ int expectedNumOfCheckpoints)
+ throws IOException {
+ Thread emptierThread = null;
+ try {
+ Runnable emptier = trash.getEmptier();
+ emptierThread = new Thread(emptier);
+ emptierThread.start();
+
+ // Shutdown the emptier thread after a given time
+ Thread.sleep(timeAlive);
+ emptierThread.interrupt();
+ emptierThread.join();
+
+ AuditableTrashPolicy at = (AuditableTrashPolicy) trash.getTrashPolicy();
+ assertEquals(
+ String.format("Expected num of checkpoints is %s, but actual is %s",
+ expectedNumOfCheckpoints, at.getNumberOfCheckpoints()),
+ expectedNumOfCheckpoints,
+ at.getNumberOfCheckpoints());
+ } catch (InterruptedException e) {
+ // Ignore
+ } finally {
+ // Avoid thread leak
+ if(emptierThread != null) {
+ emptierThread.interrupt();
+ }
+ }
}
// Test TrashPolicy. Don't care about implementation.
@@ -732,4 +951,127 @@ public class TestTrash extends TestCase {
return null;
}
}
+
+ /**
+ * A fake {@link TrashPolicy} implementation, it keeps a count
+ * on number of checkpoints in the trash. It doesn't do anything
+ * other than updating the count.
+ *
+ */
+ public static class AuditableTrashPolicy extends TrashPolicy {
+
+ public AuditableTrashPolicy() {}
+
+ public AuditableTrashPolicy(Configuration conf)
+ throws IOException {
+ this.initialize(conf, null);
+ }
+
+ @Override
+ @Deprecated
+ public void initialize(Configuration conf, FileSystem fs, Path home) {
+ this.deletionInterval = (long)(conf.getFloat(
+ FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT));
+ }
+
+ @Override
+ public void initialize(Configuration conf, FileSystem fs) {
+ this.deletionInterval = (long)(conf.getFloat(
+ FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT));
+ }
+
+ @Override
+ public boolean moveToTrash(Path path) throws IOException {
+ return false;
+ }
+
+ @Override
+ public void createCheckpoint() throws IOException {
+ AuditableCheckpoints.add();
+ }
+
+ @Override
+ public void deleteCheckpoint() throws IOException {
+ AuditableCheckpoints.delete();
+ }
+
+ @Override
+ public Path getCurrentTrashDir() {
+ return null;
+ }
+
+ @Override
+ public Runnable getEmptier() throws IOException {
+ return new AuditableEmptier(getConf());
+ }
+
+ public int getNumberOfCheckpoints() {
+ return AuditableCheckpoints.get();
+ }
+
+ /**
+ * A fake emptier that simulates to delete a checkpoint
+ * in a fixed interval.
+ */
+ private class AuditableEmptier implements Runnable {
+ private Configuration conf = null;
+ public AuditableEmptier(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public void run() {
+ AuditableTrashPolicy trash = null;
+ try {
+ trash = new AuditableTrashPolicy(conf);
+ } catch (IOException e1) {}
+ while(true) {
+ try {
+ Thread.sleep(deletionInterval);
+ trash.deleteCheckpoint();
+ } catch (IOException e) {
+ // no exception
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return true;
+ }
+ }
+
+ /**
+ * Only counts the number of checkpoints, not do anything more.
+ * Declared as an inner static class to share state between
+ * testing threads.
+ */
+ private static class AuditableCheckpoints {
+
+ private static AtomicInteger numOfCheckpoint =
+ new AtomicInteger(0);
+
+ private static void add() {
+ numOfCheckpoint.incrementAndGet();
+ System.out.println(String
+ .format("Create a checkpoint, current number of checkpoints %d",
+ numOfCheckpoint.get()));
+ }
+
+ private static void delete() {
+ if(numOfCheckpoint.get() > 0) {
+ numOfCheckpoint.decrementAndGet();
+ System.out.println(String
+ .format("Delete a checkpoint, current number of checkpoints %d",
+ numOfCheckpoint.get()));
+ }
+ }
+
+ private static int get() {
+ return numOfCheckpoint.get();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org