You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/06/12 23:49:54 UTC
hadoop git commit: HDFS-8585. Erasure Coding: Remove dataBlockNum and
parityBlockNum from StripedBlockProto. Contributed by Yi Liu.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 98d340745 -> 683332b36
HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from StripedBlockProto. Contributed by Yi Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683332b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683332b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683332b3
Branch: refs/heads/HDFS-7285
Commit: 683332b36de1040eb8901d676e666527e8c5f8fe
Parents: 98d3407
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Jun 12 14:48:53 2015 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Fri Jun 12 14:48:53 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 16 --------------
.../server/namenode/FSImageFormatPBINode.java | 23 ++++++++++----------
.../tools/offlineImageViewer/FSImageLoader.java | 20 ++---------------
.../hadoop-hdfs/src/main/proto/fsimage.proto | 2 +-
.../hadoop-hdfs/src/main/proto/hdfs.proto | 10 ---------
...TestOfflineImageViewerWithStripedBlocks.java | 14 +++++-------
7 files changed, 23 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index fa39d72..2eb8259 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -296,3 +296,6 @@
HDFS-8450. Erasure Coding: Consolidate erasure coding zone related
implementation into a single class (Rakesh R via vinayakumarb)
+
+ HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from
+ StripedBlockProto. (Yi Liu via jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 0bfc3bb..7ee6112 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -183,7 +183,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
@@ -195,7 +194,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@@ -444,20 +442,6 @@ public class PBHelper {
return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
}
- public static BlockInfoStriped convert(StripedBlockProto p, ECSchema schema) {
- return new BlockInfoStriped(convert(p.getBlock()), schema);
- }
-
- public static StripedBlockProto convert(BlockInfoStriped blk) {
- BlockProto bp = BlockProto.newBuilder().setBlockId(blk.getBlockId())
- .setGenStamp(blk.getGenerationStamp()).setNumBytes(blk.getNumBytes())
- .build();
- return StripedBlockProto.newBuilder()
- .setDataBlockNum(blk.getDataBlockNum())
- .setParityBlockNum(blk.getParityBlockNum())
- .setBlock(bp).build();
- }
-
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
BlockWithLocationsProto.Builder builder = BlockWithLocationsProto
.newBuilder().setBlock(convert(blk.getBlock()))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index e157b95..92e705f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -42,7 +41,6 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -330,10 +328,14 @@ public final class FSImageFormatPBINode {
short replication = (short) f.getReplication();
LoaderContext state = parent.getLoaderContext();
- BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.size()];
- for (int i = 0, e = bp.size(); i < e; ++i) {
- blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+ BlockInfoContiguous[] blocks = null;
+ if (!f.hasStripedBlocks()) {
+ blocks = new BlockInfoContiguous[bp.size()];
+ for (int i = 0, e = bp.size(); i < e; ++i) {
+ blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+ }
}
+
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
@@ -357,10 +359,9 @@ public final class FSImageFormatPBINode {
if (f.hasStripedBlocks()) {
// TODO: HDFS-7859
ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
- StripedBlocksFeature sb = f.getStripedBlocks();
stripeFeature = file.addStripedBlocksFeature();
- for (StripedBlockProto sp : sb.getBlocksList()) {
- stripeFeature.addBlock(PBHelper.convert(sp, schema));
+ for (BlockProto b : bp) {
+ stripeFeature.addBlock(new BlockInfoStriped(PBHelper.convert(b), schema));
}
}
@@ -658,14 +659,14 @@ public final class FSImageFormatPBINode {
FileWithStripedBlocksFeature sb = n.getStripedBlocksFeature();
if (sb != null) {
- StripedBlocksFeature.Builder builder =
- StripedBlocksFeature.newBuilder();
BlockInfoStriped[] sblocks = sb.getBlocks();
if (sblocks != null) {
for (BlockInfoStriped sblk : sblocks) {
- builder.addBlocks(PBHelper.convert(sblk));
+ b.addBlocks(PBHelper.convert(sblk));
}
}
+ StripedBlocksFeature.Builder builder =
+ StripedBlocksFeature.newBuilder();
b.setStripedBlocks(builder.build());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 42f6c0b..25d7722 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -41,15 +41,12 @@ import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
-import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.LimitInputStream;
@@ -485,21 +482,8 @@ class FSImageLoader {
static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
long size = 0;
- if (f.hasStripedBlocks()) {
- List<StripedBlockProto> blocksList = f.getStripedBlocks().getBlocksList();
- // Get total of actual data block size
- for (StripedBlockProto p : blocksList) {
- // Total usage by this striped blocks should be the total of data
- // blocks and parity blocks
- size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getBlock()
- .getNumBytes(), p.getDataBlockNum(), p.getParityBlockNum(),
- HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
- }
- } else {
- for (HdfsProtos.BlockProto p : f.getBlocksList()) {
- size += p.getNumBytes();
- }
-
+ for (HdfsProtos.BlockProto p : f.getBlocksList()) {
+ size += p.getNumBytes();
}
return size;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 3f3a71e..4d0e2dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -93,7 +93,7 @@ message INodeSection {
}
message StripedBlocksFeature {
- repeated StripedBlockProto blocks = 1;
+ // store striped blocks related information
}
message AclFeatureProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
index e6db596..dd48d7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
@@ -515,16 +515,6 @@ message BlockProto {
}
/**
- * Striped block information. Besides the basic information for a block,
- * it also contains the number of data/parity blocks.
- */
-message StripedBlockProto {
- required BlockProto block = 1;
- optional uint32 dataBlockNum = 2;
- optional uint32 parityBlockNum = 3;
-}
-
-/**
* Block and datanodes where is it located
*/
message BlockWithLocationsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683332b3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index 2a51f99..f4efbcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -136,28 +135,25 @@ public class TestOfflineImageViewerWithStripedBlocks {
}
FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath());
String fileStatus = loader.getFileStatus("/eczone/striped");
- long expectedSpaceConsumed = StripedBlockUtil.spaceConsumedByStripedBlock(
- bytes.length, HdfsConstants.NUM_DATA_BLOCKS,
- HdfsConstants.NUM_PARITY_BLOCKS, HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
+ long expectedFileSize = bytes.length;
// Verify space consumed present in BlockInfoStriped
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
- long actualSpaceConsumed = 0;
+ long actualFileSize = 0;
for (BlockInfo blockInfo : fileNode.getBlocks()) {
assertTrue("Didn't find block striped information",
blockInfo instanceof BlockInfoStriped);
- BlockInfoStriped b = (BlockInfoStriped) blockInfo;
- actualSpaceConsumed += b.spaceConsumed();
+ actualFileSize += blockInfo.getNumBytes();
}
assertEquals("Wrongly computed file size contains striped blocks",
- expectedSpaceConsumed, actualSpaceConsumed);
+ expectedFileSize, actualFileSize);
// Verify space consumed present in filestatus
String EXPECTED_FILE_SIZE = "\"length\":"
- + String.valueOf(expectedSpaceConsumed);
+ + String.valueOf(expectedFileSize);
assertTrue(
"Wrongly computed file size contains striped blocks, file status:"
+ fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,