You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2012/09/26 19:38:55 UTC
svn commit: r1390623 - in
/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common:
CHANGES.txt src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
src/test/java/org/apache/hadoop/fs/TestTrash.java
Author: jlowe
Date: Wed Sep 26 17:38:55 2012
New Revision: 1390623
URL: http://svn.apache.org/viewvc?rev=1390623&view=rev
Log:
svn merge -c 1390616 to fix HADOOP-8843. Old trash directories are never deleted on upgrade from 1.x. Contributed by Jason Lowe
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1390623&r1=1390622&r2=1390623&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Wed Sep 26 17:38:55 2012
@@ -14,6 +14,9 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
+ HADOOP-8843. Old trash directories are never deleted on upgrade
+ from 1.x (jlowe)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java?rev=1390623&r1=1390622&r2=1390623&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java Wed Sep 26 17:38:55 2012
@@ -61,6 +61,9 @@ public class TrashPolicyDefault extends
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE);
private static final DateFormat CHECKPOINT = new SimpleDateFormat("yyMMddHHmmss");
+ /** Format of checkpoint directories used prior to Hadoop 0.23. */
+ private static final DateFormat OLD_CHECKPOINT =
+ new SimpleDateFormat("yyMMddHHmm");
private static final int MSECS_PER_MINUTE = 60*1000;
private Path current;
@@ -197,9 +200,7 @@ public class TrashPolicyDefault extends
long time;
try {
- synchronized (CHECKPOINT) {
- time = CHECKPOINT.parse(name).getTime();
- }
+ time = getTimeFromCheckpoint(name);
} catch (ParseException e) {
LOG.warn("Unexpected item in trash: "+dir+". Ignoring.");
continue;
@@ -300,4 +301,22 @@ public class TrashPolicyDefault extends
return (time / interval) * interval;
}
}
+
+ private long getTimeFromCheckpoint(String name) throws ParseException {
+ long time;
+
+ try {
+ synchronized (CHECKPOINT) {
+ time = CHECKPOINT.parse(name).getTime();
+ }
+ } catch (ParseException pe) {
+ // Check for old-style checkpoint directories left over
+ // after an upgrade from Hadoop 1.x
+ synchronized (OLD_CHECKPOINT) {
+ time = OLD_CHECKPOINT.parse(name).getTime();
+ }
+ }
+
+ return time;
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java?rev=1390623&r1=1390622&r2=1390623&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java Wed Sep 26 17:38:55 2012
@@ -26,6 +26,8 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.HashSet;
import java.util.Set;
@@ -431,6 +433,36 @@ public class TestTrash extends TestCase
output.indexOf(("Consider using -skipTrash option")) != -1 );
}
+ // Verify old checkpoint format is recognized
+ {
+ // emulate two old trash checkpoint directories, one that is old enough
+ // to be deleted on the next expunge and one that isn't.
+ long trashInterval = conf.getLong(FS_TRASH_INTERVAL_KEY,
+ FS_TRASH_INTERVAL_DEFAULT);
+ long now = System.currentTimeMillis();
+ DateFormat oldCheckpointFormat = new SimpleDateFormat("yyMMddHHmm");
+ Path dirToDelete = new Path(trashRoot.getParent(),
+ oldCheckpointFormat.format(now - (trashInterval * 60 * 1000) - 1));
+ Path dirToKeep = new Path(trashRoot.getParent(),
+ oldCheckpointFormat.format(now));
+ mkdir(trashRootFs, dirToDelete);
+ mkdir(trashRootFs, dirToKeep);
+
+ // Clear out trash
+ int rc = -1;
+ try {
+ rc = shell.run(new String [] { "-expunge" } );
+ } catch (Exception e) {
+ System.err.println("Exception raised from fs expunge " +
+ e.getLocalizedMessage());
+ }
+ assertEquals(0, rc);
+ assertFalse("old checkpoint format not recognized",
+ trashRootFs.exists(dirToDelete));
+ assertTrue("old checkpoint format directory should not be removed",
+ trashRootFs.exists(dirToKeep));
+ }
+
}
public static void trashNonDefaultFS(Configuration conf) throws IOException {