You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/05/04 19:58:03 UTC
[29/50] hadoop git commit: HDFS-8145. Fix the editlog corruption
exposed by failed TestAddStripedBlocks. Contributed by Jing Zhao.
HDFS-8145. Fix the editlog corruption exposed by failed TestAddStripedBlocks. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18a2e1e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18a2e1e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18a2e1e2
Branch: refs/heads/HDFS-7285
Commit: 18a2e1e2f314be8aa00516b7779855becc565f02
Parents: 71b69ec
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Apr 17 18:13:47 2015 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon May 4 10:13:27 2015 -0700
----------------------------------------------------------------------
.../blockmanagement/BlockInfoStriped.java | 7 ------
.../namenode/ErasureCodingZoneManager.java | 12 +++++-----
.../hdfs/server/namenode/FSDirectory.java | 6 ++---
.../hdfs/server/namenode/FSEditLogLoader.java | 13 ++++++-----
.../hdfs/server/namenode/FSImageFormat.java | 4 +---
.../server/namenode/FSImageSerialization.java | 13 +++++------
.../blockmanagement/TestBlockInfoStriped.java | 23 ++++++--------------
.../hdfs/server/namenode/TestFSImage.java | 2 +-
8 files changed, 31 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 9f2f5ba..23e3153 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -244,13 +244,6 @@ public class BlockInfoStriped extends BlockInfo {
return num;
}
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeShort(dataBlockNum);
- out.writeShort(parityBlockNum);
- super.write(out);
- }
-
/**
* Convert a complete block to an under construction block.
* @return BlockInfoUnderConstruction - an under construction block.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
index 0a84083..3f94227 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
@@ -54,10 +54,6 @@ public class ErasureCodingZoneManager {
this.dir = dir;
}
- boolean getECPolicy(INodesInPath iip) throws IOException {
- return getECSchema(iip) != null;
- }
-
ECSchema getECSchema(INodesInPath iip) throws IOException {
ECZoneInfo ecZoneInfo = getECZoneInfo(iip);
return ecZoneInfo == null ? null : ecZoneInfo.getSchema();
@@ -109,7 +105,7 @@ public class ErasureCodingZoneManager {
throw new IOException("Attempt to create an erasure coding zone " +
"for a file.");
}
- if (getECPolicy(srcIIP)) {
+ if (getECSchema(srcIIP) != null) {
throw new IOException("Directory " + src + " is already in an " +
"erasure coding zone.");
}
@@ -132,8 +128,10 @@ public class ErasureCodingZoneManager {
void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
throws IOException {
assert dir.hasReadLock();
- if (getECPolicy(srcIIP)
- != getECPolicy(dstIIP)) {
+ final ECSchema srcSchema = getECSchema(srcIIP);
+ final ECSchema dstSchema = getECSchema(dstIIP);
+ if ((srcSchema != null && !srcSchema.equals(dstSchema)) ||
+ (dstSchema != null && !dstSchema.equals(srcSchema))) {
throw new IOException(
src + " can't be moved because the source and destination have " +
"different erasure coding policies.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 98b3130..5d850db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -433,7 +433,7 @@ public class FSDirectory implements Closeable {
writeLock();
try {
newiip = addINode(existing, newNode);
- if (newiip != null && getECPolicy(newiip)) {
+ if (newiip != null && isInECZone(newiip)) {
newNode.addStripedBlocksFeature();
}
} finally {
@@ -472,7 +472,7 @@ public class FSDirectory implements Closeable {
INodesInPath iip = addINode(existing, newNode);
if (iip != null) {
// check if the file is in an EC zone
- if (getECPolicy(iip)) {
+ if (isInECZone(iip)) {
newNode.addStripedBlocksFeature();
}
if (aclEntries != null) {
@@ -1413,7 +1413,7 @@ public class FSDirectory implements Closeable {
}
}
- public boolean getECPolicy(INodesInPath iip) throws IOException {
+ public boolean isInECZone(INodesInPath iip) throws IOException {
return getECSchema(iip) != null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index a0de47e..6537ab8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -418,7 +419,7 @@ public class FSEditLogLoader {
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
// TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, addCloseOp, iip, newFile, fsDir.getECPolicy(iip));
+ updateBlocks(fsDir, addCloseOp, iip, newFile, fsDir.isInECZone(iip));
break;
}
case OP_CLOSE: {
@@ -439,7 +440,7 @@ public class FSEditLogLoader {
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
// TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, addCloseOp, iip, file, fsDir.getECPolicy(iip));
+ updateBlocks(fsDir, addCloseOp, iip, file, fsDir.isInECZone(iip));
// Now close the file
if (!file.isUnderConstruction() &&
@@ -498,7 +499,7 @@ public class FSEditLogLoader {
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
// TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, updateOp, iip, oldFile, fsDir.getECPolicy(iip));
+ updateBlocks(fsDir, updateOp, iip, oldFile, fsDir.isInECZone(iip));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
@@ -516,7 +517,7 @@ public class FSEditLogLoader {
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
// TODO whether the file is striped should later be retrieved from iip
- addNewBlock(addBlockOp, oldFile, fsDir.getECPolicy(iip));
+ addNewBlock(addBlockOp, oldFile, fsDir.isInECZone(iip));
break;
}
case OP_SET_REPLICATION: {
@@ -1079,7 +1080,9 @@ public class FSEditLogLoader {
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
- newBI = new BlockInfoContiguous(newBlock, file.getBlockReplication());
+ newBI = isStriped ? new BlockInfoStriped(newBlock,
+ HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS) :
+ new BlockInfoContiguous(newBlock, file.getBlockReplication());
}
fsNamesys.getBlockManager().addBlockCollectionWithCheck(newBI, file);
file.addBlock(newBI);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index b24bdb4..205a8bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -771,10 +771,8 @@ public class FSImageFormat {
if (isStriped) {
blocks = new Block[numBlocks];
for (int j = 0; j < numBlocks; j++) {
- short dataBlockNum = in.readShort();
- short parityBlockNum = in.readShort();
blocks[j] = new BlockInfoStriped(new Block(),
- dataBlockNum, parityBlockNum);
+ HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
blocks[j].readFields(in);
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 58244e5..25febd4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
@@ -139,17 +140,15 @@ public class FSImageSerialization {
blocksStriped = new BlockInfoStriped[numBlocks];
int i = 0;
for (; i < numBlocks - 1; i++) {
- short dataBlockNum = in.readShort();
- short parityBlockNum = in.readShort();
- blocksStriped[i] = new BlockInfoStriped(new Block(), dataBlockNum,
- parityBlockNum);
+ blocksStriped[i] = new BlockInfoStriped(new Block(),
+ HdfsConstants.NUM_DATA_BLOCKS,
+ HdfsConstants.NUM_PARITY_BLOCKS);
blocksStriped[i].readFields(in);
}
if (numBlocks > 0) {
- short dataBlockNum = in.readShort();
- short parityBlockNum = in.readShort();
blocksStriped[i] = new BlockInfoStripedUnderConstruction(new Block(),
- dataBlockNum, parityBlockNum, BlockUCState.UNDER_CONSTRUCTION, null);
+ HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS,
+ BlockUCState.UNDER_CONSTRUCTION, null);
blocksStriped[i].readFields(in);
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
index c4db5d4..3b689eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
@@ -21,7 +21,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;
@@ -43,12 +42,8 @@ public class TestBlockInfoStriped {
private static final int TOTAL_NUM_BLOCKS = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
private static final long BASE_ID = -1600;
private static final Block baseBlock = new Block(BASE_ID);
- private BlockInfoStriped info;
-
- @Before
- public void setup() {
- info = new BlockInfoStriped(baseBlock, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
- }
+ private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
+ NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
private Block[] createReportedBlocks(int num) {
Block[] blocks = new Block[num];
@@ -230,17 +225,14 @@ public class TestBlockInfoStriped {
long blkID = 1;
long numBytes = 1;
long generationStamp = 1;
- short dataBlockNum = 6;
- short parityBlockNum = 3;
- ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE/Byte.SIZE*3
- + Short.SIZE/Byte.SIZE*2);
- byteBuffer.putShort(dataBlockNum).putShort(parityBlockNum)
- .putLong(blkID).putLong(numBytes).putLong(generationStamp);
+ ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE * 3);
+ byteBuffer.putLong(blkID).putLong(numBytes).putLong(generationStamp);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutput out = new DataOutputStream(byteStream);
- BlockInfoStriped blk = new BlockInfoStriped(new Block(1,1,1),
- (short)6,(short)3);
+ BlockInfoStriped blk = new BlockInfoStriped(new Block(blkID, numBytes,
+ generationStamp), NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+
try {
blk.write(out);
} catch(Exception ex) {
@@ -249,5 +241,4 @@ public class TestBlockInfoStriped {
assertEquals(byteBuffer.array().length, byteStream.toByteArray().length);
assertArrayEquals(byteBuffer.array(), byteStream.toByteArray());
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18a2e1e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 568f49d..ad8868f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -158,7 +158,7 @@ public class TestFSImage {
for (int i = 0; i < stripedBlks.length; i++) {
stripedBlks[i] = new BlockInfoStriped(
new Block(stripedBlkId + i, preferredBlockSize, timestamp),
- (short) 6, (short) 3);
+ HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
file.getStripedBlocksFeature().addBlock(stripedBlks[i]);
}