You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/01/16 19:31:21 UTC
[08/25] hadoop git commit: HDFS-5782. Change BlockListAsLongs
constructor to take Replica as parameter type instead of concrete classes
Block and ReplicaInfo. Contributed by David Powell and Joe Pallas
HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter type instead of concrete classes Block and ReplicaInfo. Contributed by David Powell and Joe Pallas
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6464a892
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6464a892
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6464a892
Branch: refs/heads/YARN-2928
Commit: 6464a8929a3623e49155febf2f9817253f9a1de3
Parents: 7fe0f25
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Wed Jan 14 13:46:06 2015 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Wed Jan 14 13:46:06 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++
.../hadoop/hdfs/protocol/BlockListAsLongs.java | 60 ++++++++++++++++----
.../server/datanode/SimulatedFSDataset.java | 4 +-
.../TestBlockHasMultipleReplicasOnSameDN.java | 2 +-
.../server/namenode/NNThroughputBenchmark.java | 2 +-
5 files changed, 57 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b879f62d..11d15f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -129,6 +129,10 @@ Trunk (Unreleased)
HDFS-6609. Use DirectorySnapshottableFeature to represent a snapshottable
directory. (Jing Zhao via wheat9)
+ HDFS-5782. Change BlockListAsLongs constructor to take Replica as parameter
+ type instead of concrete classes Block and ReplicaInfo. (David Powell
+ and Joe Pallas via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
index 8a0b731..4389714 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockListAsLongs.java
@@ -25,7 +25,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
+import org.apache.hadoop.hdfs.server.datanode.Replica;
/**
* This class provides an interface for accessing list of blocks that
@@ -85,8 +85,8 @@ public class BlockListAsLongs implements Iterable<Block> {
* @param finalized - list of finalized blocks
* @param uc - list of under construction blocks
*/
- public BlockListAsLongs(final List<? extends Block> finalized,
- final List<ReplicaInfo> uc) {
+ public BlockListAsLongs(final List<? extends Replica> finalized,
+ final List<? extends Replica> uc) {
int finalizedSize = finalized == null ? 0 : finalized.size();
int ucSize = uc == null ? 0 : uc.size();
int len = HEADER_SIZE
@@ -113,8 +113,34 @@ public class BlockListAsLongs implements Iterable<Block> {
}
}
+ /**
+ * Create block report from a list of finalized blocks. Used by
+ * NNThroughputBenchmark.
+ *
+ * @param blocks - list of finalized blocks
+ */
+ public BlockListAsLongs(final List<? extends Block> blocks) {
+ int finalizedSize = blocks == null ? 0 : blocks.size();
+ int len = HEADER_SIZE
+ + (finalizedSize + 1) * LONGS_PER_FINALIZED_BLOCK;
+
+ blockList = new long[len];
+
+ // set the header
+ blockList[0] = finalizedSize;
+ blockList[1] = 0;
+
+ // set finalized blocks
+ for (int i = 0; i < finalizedSize; i++) {
+ setBlock(i, blocks.get(i));
+ }
+
+ // set invalid delimiting block
+ setDelimitingBlock(finalizedSize);
+ }
+
public BlockListAsLongs() {
- this(null);
+ this((long[])null);
}
/**
@@ -279,18 +305,30 @@ public class BlockListAsLongs implements Iterable<Block> {
/**
* Set the indexTh block
* @param index - the index of the block to set
- * @param b - the block is set to the value of the this block
+ * @param r - the block is set to the value of the this Replica
*/
- private <T extends Block> void setBlock(final int index, final T b) {
+ private void setBlock(final int index, final Replica r) {
int pos = index2BlockId(index);
- blockList[pos] = b.getBlockId();
- blockList[pos + 1] = b.getNumBytes();
- blockList[pos + 2] = b.getGenerationStamp();
+ blockList[pos] = r.getBlockId();
+ blockList[pos + 1] = r.getNumBytes();
+ blockList[pos + 2] = r.getGenerationStamp();
if(index < getNumberOfFinalizedReplicas())
return;
- assert ((ReplicaInfo)b).getState() != ReplicaState.FINALIZED :
+ assert r.getState() != ReplicaState.FINALIZED :
"Must be under-construction replica.";
- blockList[pos + 3] = ((ReplicaInfo)b).getState().getValue();
+ blockList[pos + 3] = r.getState().getValue();
+ }
+
+ /**
+ * Set the indexTh block
+ * @param index - the index of the block to set
+ * @param b - the block is set to the value of the this Block
+ */
+ private void setBlock(final int index, final Block b) {
+ int pos = index2BlockId(index);
+ blockList[pos] = b.getBlockId();
+ blockList[pos + 1] = b.getNumBytes();
+ blockList[pos + 2] = b.getGenerationStamp();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 78eedf9..16e7a20 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -557,12 +557,12 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
synchronized BlockListAsLongs getBlockReport(String bpid) {
- final List<Block> blocks = new ArrayList<Block>();
+ final List<Replica> blocks = new ArrayList<Replica>();
final Map<Block, BInfo> map = blockMap.get(bpid);
if (map != null) {
for (BInfo b : map.values()) {
if (b.isFinalized()) {
- blocks.add(b.theBlock);
+ blocks.add(b);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
index e71c0ea..1152c74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockHasMultipleReplicasOnSameDN.java
@@ -114,7 +114,7 @@ public class TestBlockHasMultipleReplicasOnSameDN {
}
for (int i = 0; i < cluster.getStoragesPerDatanode(); ++i) {
- BlockListAsLongs bll = new BlockListAsLongs(blocks, null);
+ BlockListAsLongs bll = new BlockListAsLongs(blocks);
FsVolumeSpi v = dn.getFSDataset().getVolumes().get(i);
DatanodeStorage dns = new DatanodeStorage(v.getStorageID());
reports[i] = new StorageBlockReport(dns, bll.getBlockListAsLongs());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6464a892/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 1fb1c1f..6abe600 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -977,7 +977,7 @@ public class NNThroughputBenchmark implements Tool {
// fill remaining slots with blocks that do not exist
for(int idx = blocks.size()-1; idx >= nrBlocks; idx--)
blocks.set(idx, new Block(blocks.size() - idx, 0, 0));
- blockReportList = new BlockListAsLongs(blocks,null).getBlockListAsLongs();
+ blockReportList = new BlockListAsLongs(blocks).getBlockListAsLongs();
}
long[] getBlockReportList() {