You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by dh...@apache.org on 2010/06/20 11:08:54 UTC
svn commit: r956335 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
Author: dhruba
Date: Sun Jun 20 09:08:53 2010
New Revision: 956335
URL: http://svn.apache.org/viewvc?rev=956335&view=rev
Log:
MAPREDUCE-1492. Delete obsolete har files used on the parity files
of hdfs raid. (Rodrigo Schmidt via dhruba)
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Sun Jun 20 09:08:53 2010
@@ -102,6 +102,9 @@ Trunk (unreleased changes)
MAPREDUCE-1559. Fixes the token renewer to use the JobTracker's
credentials for talking to the NameNode. (ddas)
+ MAPREDUCE-1492. Delete obsolete har files used on the parity files
+ of hdfs raid. (Rodrigo Schmidt via dhruba)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java Sun Jun 20 09:08:53 2010
@@ -1153,6 +1153,7 @@ public class RaidNode implements RaidPro
Thread.sleep(SLEEP_TIME);
}
+ LOG.info("Started purge scan");
prevExec = now();
// fetch all categories
@@ -1235,6 +1236,12 @@ public class RaidNode implements RaidPro
// Verify if it is a har file
if (destStr.endsWith(HAR_SUFFIX)) {
+ String destParentStr = destPath.getParent().toUri().getPath();
+ String src = destParentStr.replaceFirst(destPrefix, "");
+ Path srcPath = new Path(src);
+ if (!srcFs.exists(srcPath)) {
+ destFs.delete(destPath, true);
+ }
return;
}
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java?rev=956335&r1=956334&r2=956335&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidHar.java Sun Jun 20 09:08:53 2010
@@ -198,11 +198,11 @@ public class TestRaidHar extends TestCas
" blockSize=" + blockSize + " stripeLength=" + stripeLength);
mySetup(srcPath, targetReplication, metaReplication, stripeLength);
RaidShell shell = null;
- Path dir = new Path("/user/test/raidtest/");
+ Path dir = new Path("/user/test/raidtest/subdir/");
Path file1 = new Path(dir + "/file" + iter);
RaidNode cnode = null;
try {
- Path destPath = new Path("/destraid/user/test/raidtest");
+ Path destPath = new Path("/destraid/user/test/raidtest/subdir");
fileSys.delete(dir, true);
fileSys.delete(destPath, true);
TestRaidNode.createOldFile(fileSys, file1, 1, numBlock, blockSize);
@@ -250,6 +250,26 @@ public class TestRaidHar extends TestCas
Thread.sleep(1000); // keep waiting
}
+ fileSys.delete(dir, true);
+ // wait till raid file is deleted
+ int count = 1;
+ while (count > 0) {
+ count = 0;
+ try {
+ listPaths = fileSys.listStatus(destPath);
+ if (listPaths != null) {
+ for (FileStatus s : listPaths) {
+ LOG.info("doTestHar found path " + s.getPath());
+ if (s.getPath().toString().endsWith(".har")) {
+ count++;
+ }
+ }
+ }
+ } catch (FileNotFoundException e) { } //ignoring
+ LOG.info("doTestHar waiting for har file to be deleted. Found " +
+ (listPaths == null ? "none" : listPaths.length) + " files");
+ Thread.sleep(1000);
+ }
} catch (Exception e) {
LOG.info("doTestHar Exception " + e +