You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by dh...@apache.org on 2010/02/23 08:31:08 UTC
svn commit: r915223 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Author: dhruba
Date: Tue Feb 23 07:31:08 2010
New Revision: 915223
URL: http://svn.apache.org/viewvc?rev=915223&view=rev
Log:
MAPREDUCE-1519. RaidNode fails to create new parity file
if an older parity file already exists. (Rodrigo Schmidt via dhruba)
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Feb 23 07:31:08 2010
@@ -378,6 +378,9 @@
MAPREDUCE-1378. URL encode link in jobhistory.jsp to avoid errors caused by
unescaped characters. (E. Sammer via cdouglas)
+ MAPREDUCE-1519. RaidNode fails to create new parity file
+ if an older version already exists. (Rodrigo Schmidt via dhruba)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/java/org/apache/hadoop/raid/RaidNode.java Tue Feb 23 07:31:08 2010
@@ -810,6 +810,10 @@
out.close();
out = null;
+ // delete destination if exists
+ if (outFs.exists(outpath)){
+ outFs.delete(outpath, false);
+ }
// rename tmppath to the real parity filename
if (!outFs.rename(tmppath, outpath)) {
String msg = "Unable to rename tmp file " + tmppath + " to " + outpath;
Modified: hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java?rev=915223&r1=915222&r2=915223&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/raid/src/test/org/apache/hadoop/raid/TestRaidNode.java Tue Feb 23 07:31:08 2010
@@ -334,6 +334,7 @@
long crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
FileStatus[] listPaths = null;
+ long firstmodtime = 0;
// wait till file is raided
while (true) {
Thread.sleep(20000L); // waiting
@@ -344,6 +345,7 @@
LOG.info("doCheckPolicy found path " + s.getPath());
if (!s.getPath().toString().endsWith(".tmp")) {
count++;
+ firstmodtime = s.getModificationTime();
}
}
}
@@ -354,7 +356,39 @@
(listPaths == null ? "none" : listPaths.length));
}
assertEquals(listPaths.length, 1);
- LOG.info("doCheckPolicy all files found in Raid.");
+
+ LOG.info("doCheckPolicy all files found in Raid the first time.");
+
+ LOG.info("doCheckPolicy: recreating source file");
+ crc2 = createOldFile(fileSys, file2, 3, numBlock, blockSize);
+
+ FileStatus st = fileSys.getFileStatus(file2);
+ assertTrue(st.getModificationTime() > firstmodtime);
+
+ // wait till file is raided
+ while (true) {
+ Thread.sleep(20000L); // waiting
+ listPaths = fileSys.listStatus(destPath);
+ int count = 0;
+ if (listPaths != null && listPaths.length == 1) {
+ for (FileStatus s : listPaths) {
+ LOG.info("doCheckPolicy found path " + s.getPath() + " " + s.getModificationTime());
+ if (!s.getPath().toString().endsWith(".tmp") &&
+ s.getModificationTime() > firstmodtime) {
+ count++;
+ }
+ }
+ }
+ if (count > 0) {
+ break;
+ }
+ LOG.info("doCheckPolicy waiting for files to be raided. Found " +
+ (listPaths == null ? "none" : listPaths.length));
+ }
+ assertEquals(listPaths.length, 1);
+
+ LOG.info("doCheckPolicy: file got re-raided as expected.");
+
} catch (Exception e) {
LOG.info("doCheckPolicy Exception " + e +
StringUtils.stringifyException(e));