You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/05/05 20:32:11 UTC
svn commit: r1099911 - in /hadoop/hdfs/branches/HDFS-1073:
CHANGES.HDFS-1073.txt
src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Author: todd
Date: Thu May 5 18:32:10 2011
New Revision: 1099911
URL: http://svn.apache.org/viewvc?rev=1099911&view=rev
Log:
HDFS-1892. Fix EditLogFileInputStream.getValidLength to be aware of OP_INVALID filler. Contributed by Todd Lipcon.
Modified:
hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May 5 18:32:10 2011
@@ -17,3 +17,5 @@ HDFS-1858. Add state management variable
HDFS-1859. Add some convenience functions to iterate over edit log streams
(Ivan Kelly and Todd Lipcon via todd)
HDFS-1894. Add constants for LAYOUT_VERSIONs in edits log branch (todd)
+HDFS-1892. Fix EditLogFileInputStream.getValidLength to be aware of OP_INVALID
+ filler (todd)
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java Thu May 5 18:32:10 2011
@@ -85,9 +85,10 @@ class EditLogFileInputStream extends Edi
}
/**
- * Return the length of non-zero bytes in the given file.
+ * Return the length of bytes in the given file after subtracting
+ * the trailer of 0xFF (OP_INVALID)s.
* This seeks to the end of the file and reads chunks backwards until
- * it finds a non-zero byte.
+ * it finds a non-0xFF byte.
* @param chunkSizeToRead chunk size for disk reads
* @throws IOException if the file cannot be read
*/
@@ -107,7 +108,7 @@ class EditLogFileInputStream extends Edi
int readLen = (int) Math.min(size - pos, chunkSizeToRead);
IOUtils.readFully(fis, buf, 0, readLen);
for (int i = readLen - 1; i >= 0; i--) {
- if (buf[i] != 0) {
+ if (buf[i] != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
return pos + i + 1; // + 1 since we count this byte!
}
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Thu May 5 18:32:10 2011
@@ -21,6 +21,7 @@ import junit.framework.TestCase;
import java.io.*;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -79,6 +80,8 @@ public class TestEditLog extends TestCas
"a4ff 0000 0000 0000 0000 0000 0000 0000"
).replace(" ",""));
+
+ static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode();
//
// an object that does a bunch of transactions
//
@@ -410,42 +413,43 @@ public class TestEditLog extends TestCas
assertTrue(TEST_DIR.mkdirs() || TEST_DIR.exists());
// For each of a combination of valid bytes followed by invalid,
- // make sure we determine the proper non-zero size of the file.
+ // make sure we determine the proper non-trailer size of the file.
// Sizes designed to find off-by-one errors around 1024 byte chunk size.
final int VALID_SIZES[] = new int[] {0, 1, 789, 1023, 1024, 1025};
- final int ZERO_SIZES[] = new int[] {0, 1, 1024-789-1, 1024-789,
+ final int TRAILER_SIZES[] = new int[] {0, 1, 1024-789-1, 1024-789,
1024-789+1, 1023, 1024, 1025};
for (int validPart : VALID_SIZES) {
- for (int zeroPart : ZERO_SIZES) {
- doValidLengthTest(validPart, zeroPart);
- doValidLengthTest(validPart*3, zeroPart*3);
+ for (int trailerPart : TRAILER_SIZES) {
+ doValidLengthTest(validPart, trailerPart);
+ doValidLengthTest(validPart*3, trailerPart*3);
}
}
}
- private void doValidLengthTest(int validPart, int zeroPart) throws Exception {
+ private void doValidLengthTest(int validPart, int trailerPart) throws Exception {
File file = new File(TEST_DIR, "validLengthTest");
FileOutputStream fos = new FileOutputStream(file);
try {
try {
byte[] valid = new byte[validPart];
for (int i = 0; i < validPart; i++) {
- valid[i] = (byte)(i % 10); // make sure we cycle through 0 bytes occasionally
+ valid[i] = (byte)(i & 0xFF);
}
- // The valid data shouldn't end in a 0, or else we'd think it was one shorter
+ // The valid data shouldn't end in a trailer byte, or else we'd think it was one shorter
// than actual length
if (validPart > 0) valid[validPart - 1] = 1;
fos.write(valid);
- byte[] zeros = new byte[zeroPart];
- fos.write(zeros);
+ byte[] trailer = new byte[trailerPart];
+ Arrays.fill(trailer, TRAILER_BYTE);
+ fos.write(trailer);
} finally {
fos.close();
}
long computedValid = EditLogFileInputStream.getValidLength(file, 1024);
- assertEquals("Testing valid=" + validPart + " zero=" + zeroPart,
+ assertEquals("Testing valid=" + validPart + " trailer=" + trailerPart,
validPart, computedValid);
} finally {
file.delete();