You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/19 07:15:50 UTC
[46/50] hadoop git commit: HDFS-8367 BlockInfoStriped uses EC schema.
Contributed by Kai Sasaki
HDFS-8367 BlockInfoStriped uses EC schema. Contributed by Kai Sasaki
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b596d6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b596d6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b596d6f
Branch: refs/heads/HDFS-7285
Commit: 6b596d6fb263aeb95e6a456a925bcc55670a5b88
Parents: 07a5f17
Author: Kai Zheng <ka...@intel.com>
Authored: Tue May 19 00:10:30 2015 +0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon May 18 22:11:11 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 2 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +--
.../blockmanagement/BlockInfoStriped.java | 24 ++++++++----
.../BlockInfoStripedUnderConstruction.java | 12 +++---
.../hdfs/server/namenode/FSDirWriteFileOp.java | 4 +-
.../hdfs/server/namenode/FSDirectory.java | 3 ++
.../hdfs/server/namenode/FSEditLogLoader.java | 34 +++++++++--------
.../hdfs/server/namenode/FSImageFormat.java | 10 +++--
.../server/namenode/FSImageFormatPBINode.java | 7 +++-
.../server/namenode/FSImageSerialization.java | 14 ++++---
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../blockmanagement/TestBlockInfoStriped.java | 8 +++-
.../server/namenode/TestFSEditLogLoader.java | 8 +++-
.../hdfs/server/namenode/TestFSImage.java | 6 ++-
.../server/namenode/TestStripedINodeFile.java | 39 ++++++--------------
15 files changed, 99 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 1456434..333d85f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -215,3 +215,5 @@
HDFS-8391. NN should consider current EC tasks handling count from DN while
assigning new tasks. (umamahesh)
+
+ HDFS-8367. BlockInfoStriped uses EC schema. (Kai Sasaki via Kai Zheng)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 94b2ff9..a6a356c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -203,6 +203,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand;
@@ -445,9 +446,8 @@ public class PBHelper {
return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
}
- public static BlockInfoStriped convert(StripedBlockProto p) {
- return new BlockInfoStriped(convert(p.getBlock()),
- (short) p.getDataBlockNum(), (short) p.getParityBlockNum());
+ public static BlockInfoStriped convert(StripedBlockProto p, ECSchema schema) {
+ return new BlockInfoStriped(convert(p.getBlock()), schema);
}
public static StripedBlockProto convert(BlockInfoStriped blk) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index f0e52e3..d7a48a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
@@ -39,6 +41,7 @@ import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_S
public class BlockInfoStriped extends BlockInfo {
private final short dataBlockNum;
private final short parityBlockNum;
+ private final ECSchema schema;
/**
* Always the same size with triplets. Record the block index for each triplet
* TODO: actually this is only necessary for over-replicated block. Thus can
@@ -46,16 +49,17 @@ public class BlockInfoStriped extends BlockInfo {
*/
private byte[] indices;
- public BlockInfoStriped(Block blk, short dataBlockNum, short parityBlockNum) {
- super(blk, (short) (dataBlockNum + parityBlockNum));
- indices = new byte[dataBlockNum + parityBlockNum];
+ public BlockInfoStriped(Block blk, ECSchema schema) {
+ super(blk, (short) (schema.getNumDataUnits() + schema.getNumParityUnits()));
+ indices = new byte[schema.getNumDataUnits() + schema.getNumParityUnits()];
initIndices();
- this.dataBlockNum = dataBlockNum;
- this.parityBlockNum = parityBlockNum;
+ this.schema = schema;
+ this.dataBlockNum = (short)schema.getNumDataUnits();
+ this.parityBlockNum = (short)schema.getNumParityUnits();
}
BlockInfoStriped(BlockInfoStriped b) {
- this(b, b.dataBlockNum, b.parityBlockNum);
+ this(b, b.getSchema());
this.setBlockCollection(b.getBlockCollection());
}
@@ -71,6 +75,10 @@ public class BlockInfoStriped extends BlockInfo {
return parityBlockNum;
}
+ public ECSchema getSchema() {
+ return schema;
+ }
+
private void initIndices() {
for (int i = 0; i < indices.length; i++) {
indices[i] = -1;
@@ -231,8 +239,8 @@ public class BlockInfoStriped extends BlockInfo {
BlockUCState s, DatanodeStorageInfo[] targets) {
final BlockInfoStripedUnderConstruction ucBlock;
if(isComplete()) {
- ucBlock = new BlockInfoStripedUnderConstruction(this, getDataBlockNum(),
- getParityBlockNum(), s, targets);
+ ucBlock = new BlockInfoStripedUnderConstruction(this, schema,
+ s, targets);
ucBlock.setBlockCollection(getBlockCollection());
} else {
// the block is already under construction
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
index b991615..40f880f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import java.io.IOException;
@@ -56,17 +57,16 @@ public class BlockInfoStripedUnderConstruction extends BlockInfoStriped
/**
* Constructor with null storage targets.
*/
- public BlockInfoStripedUnderConstruction(Block blk, short dataBlockNum,
- short parityBlockNum) {
- this(blk, dataBlockNum, parityBlockNum, UNDER_CONSTRUCTION, null);
+ public BlockInfoStripedUnderConstruction(Block blk, ECSchema schema) {
+ this(blk, schema, UNDER_CONSTRUCTION, null);
}
/**
* Create a striped block that is currently being constructed.
*/
- public BlockInfoStripedUnderConstruction(Block blk, short dataBlockNum,
- short parityBlockNum, BlockUCState state, DatanodeStorageInfo[] targets) {
- super(blk, dataBlockNum, parityBlockNum);
+ public BlockInfoStripedUnderConstruction(Block blk, ECSchema schema,
+ BlockUCState state, DatanodeStorageInfo[] targets) {
+ super(blk, schema);
assert getBlockUCState() != COMPLETE :
"BlockInfoStripedUnderConstruction cannot be in COMPLETE state";
this.blockUCState = state;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 324cc16..3ea6f70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -308,8 +308,8 @@ class FSDirWriteFileOp {
// check quota limits and updated space consumed
fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
numLocations, true);
- blockInfo = new BlockInfoStripedUnderConstruction(block, numDataUnits,
- numParityUnits, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
+ blockInfo = new BlockInfoStripedUnderConstruction(block, ecSchema,
+ HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
targets);
} else {
// check quota limits and updated space consumed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index f879fb9..68fa8b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -57,6 +57,9 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 0415eb5..7cbf34d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress.Counter;
import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step;
import org.apache.hadoop.hdfs.util.Holder;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.ChunkedArrayList;
import com.google.common.base.Joiner;
@@ -419,8 +420,8 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- // TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, addCloseOp, iip, newFile, fsDir.isInECZone(iip));
+ updateBlocks(fsDir, addCloseOp, iip, newFile,
+ fsDir.getECSchema(iip), fsDir.isInECZone(iip));
break;
}
case OP_CLOSE: {
@@ -440,8 +441,8 @@ public class FSEditLogLoader {
// Update the salient file attributes.
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- // TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, addCloseOp, iip, file, fsDir.isInECZone(iip));
+ updateBlocks(fsDir, addCloseOp, iip, file,
+ fsDir.getECSchema(iip), fsDir.isInECZone(iip));
// Now close the file
if (!file.isUnderConstruction() &&
@@ -499,9 +500,9 @@ public class FSEditLogLoader {
INodesInPath iip = fsDir.getINodesInPath(path, true);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
- // TODO whether the file is striped should later be retrieved from iip
- updateBlocks(fsDir, updateOp, iip, oldFile, fsDir.isInECZone(iip));
-
+ updateBlocks(fsDir, updateOp, iip, oldFile,
+ fsDir.getECSchema(iip), fsDir.isInECZone(iip));
+
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
}
@@ -517,8 +518,8 @@ public class FSEditLogLoader {
INodesInPath iip = fsDir.getINodesInPath(path, true);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
- // TODO whether the file is striped should later be retrieved from iip
- addNewBlock(addBlockOp, oldFile, fsDir.isInECZone(iip));
+ addNewBlock(addBlockOp, oldFile,
+ fsDir.getECSchema(iip), fsDir.isInECZone(iip));
break;
}
case OP_SET_REPLICATION: {
@@ -959,7 +960,8 @@ public class FSEditLogLoader {
/**
* Add a new block into the given INodeFile
*/
- private void addNewBlock(AddBlockOp op, INodeFile file, boolean isStriped)
+ private void addNewBlock(AddBlockOp op, INodeFile file,
+ ECSchema schema, boolean isStriped)
throws IOException {
BlockInfo[] oldBlocks = file.getBlocks();
Block pBlock = op.getPenultimateBlock();
@@ -988,8 +990,7 @@ public class FSEditLogLoader {
// add the new block
final BlockInfo newBlockInfo;
if (isStriped) {
- newBlockInfo = new BlockInfoStripedUnderConstruction(newBlock,
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
+ newBlockInfo = new BlockInfoStripedUnderConstruction(newBlock, schema);
} else {
newBlockInfo = new BlockInfoContiguousUnderConstruction(newBlock,
file.getPreferredBlockReplication());
@@ -1004,7 +1005,8 @@ public class FSEditLogLoader {
* @throws IOException
*/
private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
- INodesInPath iip, INodeFile file, boolean isStriped) throws IOException {
+ INodesInPath iip, INodeFile file, ECSchema schema,
+ boolean isStriped) throws IOException {
// Update its block list
BlockInfo[] oldBlocks = file.getBlocks();
Block[] newBlocks = op.getBlocks();
@@ -1072,8 +1074,7 @@ public class FSEditLogLoader {
// what about an old-version fsync() where fsync isn't called
// until several blocks in?
if (isStriped) {
- newBI = new BlockInfoStripedUnderConstruction(newBlock,
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
+ newBI = new BlockInfoStripedUnderConstruction(newBlock, schema);
} else {
newBI = new BlockInfoContiguousUnderConstruction(newBlock,
file.getPreferredBlockReplication());
@@ -1083,8 +1084,9 @@ public class FSEditLogLoader {
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
+ // TODO: ECSchema can be restored from persisted file (HDFS-7859).
newBI = isStriped ? new BlockInfoStriped(newBlock,
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS) :
+ ErasureCodingSchemaManager.getSystemDefaultSchema()) :
new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication());
}
fsNamesys.getBlockManager().addBlockCollectionWithCheck(newBI, file);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 6f485f5..3083952 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.StringUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -762,6 +763,9 @@ public class FSImageFormat {
NameNodeLayoutVersion.Feature.ERASURE_CODING, imgVersion)
&& (in.readBoolean());
final int numBlocks = in.readInt();
+ // TODO: ECSchema can be restored from persisted file (HDFS-7859).
+ final ECSchema schema = isStriped ?
+ ErasureCodingSchemaManager.getSystemDefaultSchema() : null;
if (numBlocks >= 0) {
// file
@@ -771,8 +775,7 @@ public class FSImageFormat {
if (isStriped) {
blocks = new Block[numBlocks];
for (int j = 0; j < numBlocks; j++) {
- blocks[j] = new BlockInfoStriped(new Block(),
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
+ blocks[j] = new BlockInfoStriped(new Block(), schema);
blocks[j].readFields(in);
}
} else {
@@ -804,8 +807,7 @@ public class FSImageFormat {
BlockInfoStriped lastStripedBlk = (BlockInfoStriped) lastBlk;
blocks[blocks.length - 1]
= new BlockInfoStripedUnderConstruction(lastBlk,
- lastStripedBlk.getDataBlockNum(),
- lastStripedBlk.getParityBlockNum());
+ lastStripedBlk.getSchema());
} else {
blocks[blocks.length - 1]
= new BlockInfoContiguousUnderConstruction(lastBlk,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 583f4b0..e157b95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.erasurecode.ECSchema;
@InterfaceAudience.Private
public final class FSImageFormatPBINode {
@@ -354,10 +355,12 @@ public final class FSImageFormatPBINode {
FileWithStripedBlocksFeature stripeFeature = null;
if (f.hasStripedBlocks()) {
+ // TODO: HDFS-7859
+ ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
StripedBlocksFeature sb = f.getStripedBlocks();
stripeFeature = file.addStripedBlocksFeature();
for (StripedBlockProto sp : sb.getBlocksList()) {
- stripeFeature.addBlock(PBHelper.convert(sp));
+ stripeFeature.addBlock(PBHelper.convert(sp, schema));
}
}
@@ -372,7 +375,7 @@ public final class FSImageFormatPBINode {
if (stripeFeature != null) {
BlockInfoStriped striped = (BlockInfoStriped) lastBlk;
ucBlk = new BlockInfoStripedUnderConstruction(striped,
- striped.getDataBlockNum(), striped.getParityBlockNum());
+ striped.getSchema());
} else {
ucBlk = new BlockInfoContiguousUnderConstruction(lastBlk,
replication);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 25febd4..4744716 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
@@ -130,7 +131,11 @@ public class FSImageSerialization {
final boolean isStriped = NameNodeLayoutVersion.supports(
NameNodeLayoutVersion.Feature.ERASURE_CODING, imgVersion)
&& (in.readBoolean());
-
+
+ // TODO: ECSchema can be restored from persisted file (HDFS-7859).
+ final ECSchema schema = isStriped ?
+ ErasureCodingSchemaManager.getSystemDefaultSchema() : null;
+
int numBlocks = in.readInt();
final BlockInfoContiguous[] blocksContiguous;
@@ -140,15 +145,12 @@ public class FSImageSerialization {
blocksStriped = new BlockInfoStriped[numBlocks];
int i = 0;
for (; i < numBlocks - 1; i++) {
- blocksStriped[i] = new BlockInfoStriped(new Block(),
- HdfsConstants.NUM_DATA_BLOCKS,
- HdfsConstants.NUM_PARITY_BLOCKS);
+ blocksStriped[i] = new BlockInfoStriped(new Block(), schema);
blocksStriped[i].readFields(in);
}
if (numBlocks > 0) {
blocksStriped[i] = new BlockInfoStripedUnderConstruction(new Block(),
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS,
- BlockUCState.UNDER_CONSTRUCTION, null);
+ schema, BlockUCState.UNDER_CONSTRUCTION, null);
blocksStriped[i].readFields(in);
}
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 322838b..fd4039e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3872,7 +3872,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
fileINode.getPreferredBlockSize() * numBlocks;
final BlockInfoStriped striped = new BlockInfoStriped(commitBlock,
- numDataUnits, numParityUnits);
+ ecSchema);
final long actualBlockGroupSize = striped.spaceConsumed();
diff = fullBlockGroupSize - actualBlockGroupSize;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
index 3b689eb..2d6b509 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;
@@ -42,8 +44,10 @@ public class TestBlockInfoStriped {
private static final int TOTAL_NUM_BLOCKS = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
private static final long BASE_ID = -1600;
private static final Block baseBlock = new Block(BASE_ID);
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
- NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+ testSchema);
private Block[] createReportedBlocks(int num) {
Block[] blocks = new Block[num];
@@ -231,7 +235,7 @@ public class TestBlockInfoStriped {
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutput out = new DataOutputStream(byteStream);
BlockInfoStriped blk = new BlockInfoStriped(new Block(blkID, numBytes,
- generationStamp), NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+ generationStamp), testSchema);
try {
blk.write(out);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index 57f62c9..2909423 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;
@@ -72,6 +73,9 @@ public class TestFSEditLogLoader {
private static final File TEST_DIR = PathUtils.getTestDir(TestFSEditLogLoader.class);
private static final int NUM_DATA_NODES = 0;
+
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
@Test
public void testDisplayRecentEditLogOpCodes() throws IOException {
@@ -459,7 +463,7 @@ public class TestFSEditLogLoader {
// Add a striped block to the file
BlockInfoStriped stripedBlk = new BlockInfoStriped(
- new Block(blkId, blkNumBytes, timestamp), blockNum, parityNum);
+ new Block(blkId, blkNumBytes, timestamp), testSchema);
INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
file.toUnderConstruction(clientName, clientMachine);
file.getStripedBlocksFeature().addBlock(stripedBlk);
@@ -525,7 +529,7 @@ public class TestFSEditLogLoader {
Path p = new Path(testFilePath);
DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
BlockInfoStriped stripedBlk = new BlockInfoStriped(
- new Block(blkId, blkNumBytes, timestamp), blockNum, parityNum);
+ new Block(blkId, blkNumBytes, timestamp), testSchema);
INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
file.toUnderConstruction(clientName, clientMachine);
file.getStripedBlocksFeature().addBlock(stripedBlk);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index bb37534..5e015bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.Assert;
import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -67,6 +68,9 @@ public class TestFSImage {
private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
"image-with-zero-block-size.tar.gz";
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
+
@Test
public void testPersist() throws IOException {
Configuration conf = new Configuration();
@@ -158,7 +162,7 @@ public class TestFSImage {
for (int i = 0; i < stripedBlks.length; i++) {
stripedBlks[i] = new BlockInfoStriped(
new Block(stripedBlkId + i, preferredBlockSize, timestamp),
- HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
+ testSchema);
file.getStripedBlocksFeature().addBlock(stripedBlks[i]);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b596d6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index e24bc54..9e4da84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -51,6 +51,9 @@ public class TestStripedINodeFile {
private final BlockStoragePolicy defaultPolicy =
defaultSuite.getDefaultPolicy();
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
+
private static INodeFile createStripedINodeFile() {
return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
null, (short)0, 1024L, HdfsServerConstants.COLD_STORAGE_POLICY_ID);
@@ -69,9 +72,7 @@ public class TestStripedINodeFile {
ECSchema defaultSchema = ErasureCodingSchemaManager.getSystemDefaultSchema();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
- = new BlockInfoStriped(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk, testSchema);
assertEquals(9, blockInfoStriped.getTotalBlockNum());
}
@@ -83,9 +84,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
- = new BlockInfoStriped(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk, testSchema);
inf.addBlock(blockInfoStriped);
assertEquals(1, inf.getBlocks().length);
}
@@ -98,9 +97,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
- = new BlockInfoStriped(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk, testSchema);
blockInfoStriped.setNumBytes(1);
inf.addBlock(blockInfoStriped);
// 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
@@ -127,15 +124,11 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk1 = new Block(1);
BlockInfoStriped blockInfoStriped1
- = new BlockInfoStriped(blk1,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk1, testSchema);
blockInfoStriped1.setNumBytes(1);
Block blk2 = new Block(2);
BlockInfoStriped blockInfoStriped2
- = new BlockInfoStriped(blk2,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk2, testSchema);
blockInfoStriped2.setNumBytes(1);
inf.addBlock(blockInfoStriped1);
inf.addBlock(blockInfoStriped2);
@@ -152,9 +145,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
- = new BlockInfoStriped(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk, testSchema);
blockInfoStriped.setNumBytes(100);
inf.addBlock(blockInfoStriped);
// Compute file size should return actual data
@@ -171,9 +162,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStripedUnderConstruction bInfoStripedUC
- = new BlockInfoStripedUnderConstruction(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStripedUnderConstruction(blk, testSchema);
bInfoStripedUC.setNumBytes(100);
inf.addBlock(bInfoStripedUC);
assertEquals(100, inf.computeFileSize());
@@ -188,9 +177,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
- = new BlockInfoStriped(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStriped(blk, testSchema);
blockInfoStriped.setNumBytes(100);
inf.addBlock(blockInfoStriped);
@@ -213,9 +200,7 @@ public class TestStripedINodeFile {
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStripedUnderConstruction bInfoStripedUC
- = new BlockInfoStripedUnderConstruction(blk,
- (short)defaultSchema.getNumDataUnits(),
- (short)defaultSchema.getNumParityUnits());
+ = new BlockInfoStripedUnderConstruction(blk, testSchema);
bInfoStripedUC.setNumBytes(100);
inf.addBlock(bInfoStripedUC);