You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/03 01:47:37 UTC
[05/50] [abbrv] hadoop git commit: HDFS-8482. Rename
BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
HDFS-8482. Rename BlockInfoContiguous to BlockInfo. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b88df72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b88df72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b88df72
Branch: refs/heads/YARN-2928
Commit: 8b88df723b75e8a81ed7db7599baeb4098059f46
Parents: 00c4b9d
Author: Andrew Wang <wa...@apache.org>
Authored: Wed May 27 15:37:01 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Jun 2 15:46:59 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../server/blockmanagement/BlockCollection.java | 8 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 399 +++++++++++++++++++
.../blockmanagement/BlockInfoContiguous.java | 399 -------------------
.../BlockInfoContiguousUnderConstruction.java | 6 +-
.../server/blockmanagement/BlockManager.java | 124 +++---
.../hdfs/server/blockmanagement/BlocksMap.java | 32 +-
.../CacheReplicationMonitor.java | 14 +-
.../blockmanagement/DatanodeDescriptor.java | 16 +-
.../blockmanagement/DatanodeStorageInfo.java | 25 +-
.../blockmanagement/DecommissionManager.java | 22 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 14 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 8 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 14 +-
.../hdfs/server/namenode/FSImageFormat.java | 14 +-
.../server/namenode/FSImageFormatPBINode.java | 11 +-
.../server/namenode/FSImageSerialization.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 42 +-
.../namenode/FileUnderConstructionFeature.java | 6 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 74 ++--
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../snapshot/FSImageFormatPBSnapshot.java | 8 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 10 +-
.../server/namenode/snapshot/FileDiffList.java | 20 +-
.../snapshot/FileWithSnapshotFeature.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +-
.../apache/hadoop/hdfs/TestDecommission.java | 4 +-
.../server/blockmanagement/TestBlockInfo.java | 20 +-
.../blockmanagement/TestBlockManager.java | 36 +-
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestPendingReplication.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 9 +-
.../hdfs/server/namenode/CreateEditsLog.java | 8 +-
.../hdfs/server/namenode/TestAddBlock.java | 12 +-
.../namenode/TestBlockUnderConstruction.java | 6 +-
.../TestCommitBlockSynchronization.java | 8 +-
.../hdfs/server/namenode/TestEditLog.java | 4 +-
.../hdfs/server/namenode/TestFSImage.java | 4 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../server/namenode/TestGetBlockLocations.java | 4 +-
.../hdfs/server/namenode/TestINodeFile.java | 4 +-
.../namenode/TestTruncateQuotaUpdate.java | 16 +-
.../namenode/snapshot/SnapshotTestHelper.java | 4 +-
.../snapshot/TestFileWithSnapshotFeature.java | 6 +-
.../snapshot/TestSnapshotBlocksMap.java | 24 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
48 files changed, 745 insertions(+), 750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ce97552..948a516 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -580,6 +580,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8377. Support HTTP/2 in datanode. (Duo Zhang via wheat9)
+ HDFS-8482. Rename BlockInfoContiguous to BlockInfo. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index c0a959c..02a1d05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -31,7 +31,7 @@ public interface BlockCollection {
/**
* Get the last block of the collection.
*/
- public BlockInfoContiguous getLastBlock();
+ public BlockInfo getLastBlock();
/**
* Get content summary.
@@ -46,7 +46,7 @@ public interface BlockCollection {
/**
* Get the blocks.
*/
- public BlockInfoContiguous[] getBlocks();
+ public BlockInfo[] getBlocks();
/**
* Get preferred block size for the collection
@@ -73,13 +73,13 @@ public interface BlockCollection {
/**
* Set the block at the given index.
*/
- public void setBlock(int index, BlockInfoContiguous blk);
+ public void setBlock(int index, BlockInfo blk);
/**
* Convert the last block of the collection to an under-construction block
* and set the locations.
*/
- public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfoContiguous lastBlock,
+ public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfo lastBlock,
DatanodeStorageInfo[] targets) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
new file mode 100644
index 0000000..0370c57
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -0,0 +1,399 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.util.LightWeightGSet;
+
+/**
+ * BlockInfo class maintains for a given block
+ * the {@link BlockCollection} it is part of and datanodes where the replicas of
+ * the block are stored.
+ */
+@InterfaceAudience.Private
+public class BlockInfo extends Block
+ implements LightWeightGSet.LinkedElement {
+ public static final BlockInfo[] EMPTY_ARRAY = {};
+
+ private BlockCollection bc;
+
+ /** For implementing {@link LightWeightGSet.LinkedElement} interface */
+ private LightWeightGSet.LinkedElement nextLinkedElement;
+
+ /**
+ * This array contains triplets of references. For each i-th storage, the
+ * block belongs to triplets[3*i] is the reference to the
+ * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
+ * references to the previous and the next blocks, respectively, in the list
+ * of blocks belonging to this storage.
+ *
+ * Using previous and next in Object triplets is done instead of a
+ * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
+ * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
+ * bytes using the triplets.
+ */
+ private Object[] triplets;
+
+ /**
+ * Construct an entry for blocksmap
+ * @param replication the block's replication factor
+ */
+ public BlockInfo(short replication) {
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ public BlockInfo(Block blk, short replication) {
+ super(blk);
+ this.triplets = new Object[3*replication];
+ this.bc = null;
+ }
+
+ /**
+ * Copy construction.
+ * This is used to convert BlockInfoUnderConstruction
+ * @param from BlockInfo to copy from.
+ */
+ protected BlockInfo(BlockInfo from) {
+ super(from);
+ this.triplets = new Object[from.triplets.length];
+ this.bc = from.bc;
+ }
+
+ public BlockCollection getBlockCollection() {
+ return bc;
+ }
+
+ public void setBlockCollection(BlockCollection bc) {
+ this.bc = bc;
+ }
+
+ public boolean isDeleted() {
+ return (bc == null);
+ }
+
+ public DatanodeDescriptor getDatanode(int index) {
+ DatanodeStorageInfo storage = getStorageInfo(index);
+ return storage == null ? null : storage.getDatanodeDescriptor();
+ }
+
+ DatanodeStorageInfo getStorageInfo(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ return (DatanodeStorageInfo)triplets[index*3];
+ }
+
+ private BlockInfo getPrevious(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+1];
+ assert info == null ||
+ info.getClass().getName().startsWith(BlockInfo.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ BlockInfo getNext(int index) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+2];
+ assert info == null || info.getClass().getName().startsWith(
+ BlockInfo.class.getName()) :
+ "BlockInfo is expected at " + index*3;
+ return info;
+ }
+
+ private void setStorageInfo(int index, DatanodeStorageInfo storage) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
+ triplets[index*3] = storage;
+ }
+
+ /**
+ * Return the previous block on the block list for the datanode at
+ * position index. Set the previous block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to previous on the list of blocks
+ * @return current previous block on the list of blocks
+ */
+ private BlockInfo setPrevious(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+1];
+ triplets[index*3+1] = to;
+ return info;
+ }
+
+ /**
+ * Return the next block on the block list for the datanode at
+ * position index. Set the next block on the list to "to".
+ *
+ * @param index - the datanode index
+ * @param to - block to be set to next on the list of blocks
+ * * @return current next block on the list of blocks
+ */
+ private BlockInfo setNext(int index, BlockInfo to) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
+ BlockInfo info = (BlockInfo)triplets[index*3+2];
+ triplets[index*3+2] = to;
+ return info;
+ }
+
+ public int getCapacity() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ return triplets.length / 3;
+ }
+
+ /**
+ * Ensure that there is enough space to include num more triplets.
+ * @return first free triplet index.
+ */
+ private int ensureCapacity(int num) {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ int last = numNodes();
+ if(triplets.length >= (last+num)*3)
+ return last;
+ /* Not enough space left. Create a new array. Should normally
+ * happen only when replication is manually increased by the user. */
+ Object[] old = triplets;
+ triplets = new Object[(last+num)*3];
+ System.arraycopy(old, 0, triplets, 0, last*3);
+ return last;
+ }
+
+ /**
+ * Count the number of data-nodes the block belongs to.
+ */
+ public int numNodes() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ for(int idx = getCapacity()-1; idx >= 0; idx--) {
+ if(getDatanode(idx) != null)
+ return idx+1;
+ }
+ return 0;
+ }
+
+ /**
+ * Add a {@link DatanodeStorageInfo} location for a block
+ */
+ boolean addStorage(DatanodeStorageInfo storage) {
+ // find the last null node
+ int lastNode = ensureCapacity(1);
+ setStorageInfo(lastNode, storage);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Remove {@link DatanodeStorageInfo} location for a block
+ */
+ boolean removeStorage(DatanodeStorageInfo storage) {
+ int dnIndex = findStorageInfo(storage);
+ if(dnIndex < 0) // the node is not found
+ return false;
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is still in the list and must be removed first.";
+ // find the last not null node
+ int lastNode = numNodes()-1;
+ // replace current node triplet by the lastNode one
+ setStorageInfo(dnIndex, getStorageInfo(lastNode));
+ setNext(dnIndex, getNext(lastNode));
+ setPrevious(dnIndex, getPrevious(lastNode));
+ // set the last triplet to null
+ setStorageInfo(lastNode, null);
+ setNext(lastNode, null);
+ setPrevious(lastNode, null);
+ return true;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return DatanodeStorageInfo or null if not found.
+ */
+ DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if(cur == null)
+ break;
+ if(cur.getDatanodeDescriptor() == dn)
+ return cur;
+ }
+ return null;
+ }
+
+ /**
+ * Find specified DatanodeStorageInfo.
+ * @return index or -1 if not found.
+ */
+ int findStorageInfo(DatanodeStorageInfo storageInfo) {
+ int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if (cur == storageInfo) {
+ return idx;
+ }
+ if (cur == null) {
+ break;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Insert this block into the head of the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If the head is null then form a new list.
+ * @return current block as the new head of the list.
+ */
+ BlockInfo listInsert(BlockInfo head,
+ DatanodeStorageInfo storage) {
+ int dnIndex = this.findStorageInfo(storage);
+ assert dnIndex >= 0 : "Data node is not found: current";
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is already in the list and cannot be inserted.";
+ this.setPrevious(dnIndex, null);
+ this.setNext(dnIndex, head);
+ if(head != null)
+ head.setPrevious(head.findStorageInfo(storage), this);
+ return this;
+ }
+
+ /**
+ * Remove this block from the list of blocks
+ * related to the specified DatanodeStorageInfo.
+ * If this block is the head of the list then return the next block as
+ * the new head.
+ * @return the new head of the list or null if the list becomes
+ * empy after deletion.
+ */
+ BlockInfo listRemove(BlockInfo head,
+ DatanodeStorageInfo storage) {
+ if(head == null)
+ return null;
+ int dnIndex = this.findStorageInfo(storage);
+ if(dnIndex < 0) // this block is not on the data-node list
+ return head;
+
+ BlockInfo next = this.getNext(dnIndex);
+ BlockInfo prev = this.getPrevious(dnIndex);
+ this.setNext(dnIndex, null);
+ this.setPrevious(dnIndex, null);
+ if(prev != null)
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if(next != null)
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ if(this == head) // removing the head
+ head = next;
+ return head;
+ }
+
+ /**
+ * Remove this block from the list of blocks related to the specified
+ * DatanodeDescriptor. Insert it into the head of the list of blocks.
+ *
+ * @return the new head of the list.
+ */
+ public BlockInfo moveBlockToHead(BlockInfo head,
+ DatanodeStorageInfo storage, int curIndex, int headIndex) {
+ if (head == this) {
+ return this;
+ }
+ BlockInfo next = this.setNext(curIndex, head);
+ BlockInfo prev = this.setPrevious(curIndex, null);
+
+ head.setPrevious(headIndex, this);
+ prev.setNext(prev.findStorageInfo(storage), next);
+ if (next != null) {
+ next.setPrevious(next.findStorageInfo(storage), prev);
+ }
+ return this;
+ }
+
+ /**
+ * BlockInfo represents a block that is not being constructed.
+ * In order to start modifying the block, the BlockInfo should be converted
+ * to {@link BlockInfoContiguousUnderConstruction}.
+ * @return {@link BlockUCState#COMPLETE}
+ */
+ public BlockUCState getBlockUCState() {
+ return BlockUCState.COMPLETE;
+ }
+
+ /**
+ * Is this block complete?
+ *
+ * @return true if the state of the block is {@link BlockUCState#COMPLETE}
+ */
+ public boolean isComplete() {
+ return getBlockUCState().equals(BlockUCState.COMPLETE);
+ }
+
+ /**
+ * Convert a complete block to an under construction block.
+ * @return BlockInfoUnderConstruction - an under construction block.
+ */
+ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
+ BlockUCState s, DatanodeStorageInfo[] targets) {
+ if(isComplete()) {
+ BlockInfoContiguousUnderConstruction ucBlock =
+ new BlockInfoContiguousUnderConstruction(this,
+ getBlockCollection().getPreferredBlockReplication(), s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+ // the block is already under construction
+ BlockInfoContiguousUnderConstruction ucBlock =
+ (BlockInfoContiguousUnderConstruction)this;
+ ucBlock.setBlockUCState(s);
+ ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+
+ @Override
+ public int hashCode() {
+ // Super implementation is sufficient
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public LightWeightGSet.LinkedElement getNext() {
+ return nextLinkedElement;
+ }
+
+ @Override
+ public void setNext(LightWeightGSet.LinkedElement next) {
+ this.nextLinkedElement = next;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
deleted file mode 100644
index 769046b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.blockmanagement;
-
-import java.util.LinkedList;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.util.LightWeightGSet;
-
-/**
- * BlockInfo class maintains for a given block
- * the {@link BlockCollection} it is part of and datanodes where the replicas of
- * the block are stored.
- */
-@InterfaceAudience.Private
-public class BlockInfoContiguous extends Block
- implements LightWeightGSet.LinkedElement {
- public static final BlockInfoContiguous[] EMPTY_ARRAY = {};
-
- private BlockCollection bc;
-
- /** For implementing {@link LightWeightGSet.LinkedElement} interface */
- private LightWeightGSet.LinkedElement nextLinkedElement;
-
- /**
- * This array contains triplets of references. For each i-th storage, the
- * block belongs to triplets[3*i] is the reference to the
- * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
- * references to the previous and the next blocks, respectively, in the list
- * of blocks belonging to this storage.
- *
- * Using previous and next in Object triplets is done instead of a
- * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
- * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
- * bytes using the triplets.
- */
- private Object[] triplets;
-
- /**
- * Construct an entry for blocksmap
- * @param replication the block's replication factor
- */
- public BlockInfoContiguous(short replication) {
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- public BlockInfoContiguous(Block blk, short replication) {
- super(blk);
- this.triplets = new Object[3*replication];
- this.bc = null;
- }
-
- /**
- * Copy construction.
- * This is used to convert BlockInfoUnderConstruction
- * @param from BlockInfo to copy from.
- */
- protected BlockInfoContiguous(BlockInfoContiguous from) {
- super(from);
- this.triplets = new Object[from.triplets.length];
- this.bc = from.bc;
- }
-
- public BlockCollection getBlockCollection() {
- return bc;
- }
-
- public void setBlockCollection(BlockCollection bc) {
- this.bc = bc;
- }
-
- public boolean isDeleted() {
- return (bc == null);
- }
-
- public DatanodeDescriptor getDatanode(int index) {
- DatanodeStorageInfo storage = getStorageInfo(index);
- return storage == null ? null : storage.getDatanodeDescriptor();
- }
-
- DatanodeStorageInfo getStorageInfo(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- return (DatanodeStorageInfo)triplets[index*3];
- }
-
- private BlockInfoContiguous getPrevious(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
- assert info == null ||
- info.getClass().getName().startsWith(BlockInfoContiguous.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- BlockInfoContiguous getNext(int index) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
- assert info == null || info.getClass().getName().startsWith(
- BlockInfoContiguous.class.getName()) :
- "BlockInfo is expected at " + index*3;
- return info;
- }
-
- private void setStorageInfo(int index, DatanodeStorageInfo storage) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3 < triplets.length : "Index is out of bound";
- triplets[index*3] = storage;
- }
-
- /**
- * Return the previous block on the block list for the datanode at
- * position index. Set the previous block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to previous on the list of blocks
- * @return current previous block on the list of blocks
- */
- private BlockInfoContiguous setPrevious(int index, BlockInfoContiguous to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+1 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+1];
- triplets[index*3+1] = to;
- return info;
- }
-
- /**
- * Return the next block on the block list for the datanode at
- * position index. Set the next block on the list to "to".
- *
- * @param index - the datanode index
- * @param to - block to be set to next on the list of blocks
- * * @return current next block on the list of blocks
- */
- private BlockInfoContiguous setNext(int index, BlockInfoContiguous to) {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert index >= 0 && index*3+2 < triplets.length : "Index is out of bound";
- BlockInfoContiguous info = (BlockInfoContiguous)triplets[index*3+2];
- triplets[index*3+2] = to;
- return info;
- }
-
- public int getCapacity() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- return triplets.length / 3;
- }
-
- /**
- * Ensure that there is enough space to include num more triplets.
- * @return first free triplet index.
- */
- private int ensureCapacity(int num) {
- assert this.triplets != null : "BlockInfo is not initialized";
- int last = numNodes();
- if(triplets.length >= (last+num)*3)
- return last;
- /* Not enough space left. Create a new array. Should normally
- * happen only when replication is manually increased by the user. */
- Object[] old = triplets;
- triplets = new Object[(last+num)*3];
- System.arraycopy(old, 0, triplets, 0, last*3);
- return last;
- }
-
- /**
- * Count the number of data-nodes the block belongs to.
- */
- public int numNodes() {
- assert this.triplets != null : "BlockInfo is not initialized";
- assert triplets.length % 3 == 0 : "Malformed BlockInfo";
- for(int idx = getCapacity()-1; idx >= 0; idx--) {
- if(getDatanode(idx) != null)
- return idx+1;
- }
- return 0;
- }
-
- /**
- * Add a {@link DatanodeStorageInfo} location for a block
- */
- boolean addStorage(DatanodeStorageInfo storage) {
- // find the last null node
- int lastNode = ensureCapacity(1);
- setStorageInfo(lastNode, storage);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Remove {@link DatanodeStorageInfo} location for a block
- */
- boolean removeStorage(DatanodeStorageInfo storage) {
- int dnIndex = findStorageInfo(storage);
- if(dnIndex < 0) // the node is not found
- return false;
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is still in the list and must be removed first.";
- // find the last not null node
- int lastNode = numNodes()-1;
- // replace current node triplet by the lastNode one
- setStorageInfo(dnIndex, getStorageInfo(lastNode));
- setNext(dnIndex, getNext(lastNode));
- setPrevious(dnIndex, getPrevious(lastNode));
- // set the last triplet to null
- setStorageInfo(lastNode, null);
- setNext(lastNode, null);
- setPrevious(lastNode, null);
- return true;
- }
-
- /**
- * Find specified DatanodeStorageInfo.
- * @return DatanodeStorageInfo or null if not found.
- */
- DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if(cur == null)
- break;
- if(cur.getDatanodeDescriptor() == dn)
- return cur;
- }
- return null;
- }
-
- /**
- * Find specified DatanodeStorageInfo.
- * @return index or -1 if not found.
- */
- int findStorageInfo(DatanodeStorageInfo storageInfo) {
- int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
- DatanodeStorageInfo cur = getStorageInfo(idx);
- if (cur == storageInfo) {
- return idx;
- }
- if (cur == null) {
- break;
- }
- }
- return -1;
- }
-
- /**
- * Insert this block into the head of the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If the head is null then form a new list.
- * @return current block as the new head of the list.
- */
- BlockInfoContiguous listInsert(BlockInfoContiguous head,
- DatanodeStorageInfo storage) {
- int dnIndex = this.findStorageInfo(storage);
- assert dnIndex >= 0 : "Data node is not found: current";
- assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
- "Block is already in the list and cannot be inserted.";
- this.setPrevious(dnIndex, null);
- this.setNext(dnIndex, head);
- if(head != null)
- head.setPrevious(head.findStorageInfo(storage), this);
- return this;
- }
-
- /**
- * Remove this block from the list of blocks
- * related to the specified DatanodeStorageInfo.
- * If this block is the head of the list then return the next block as
- * the new head.
- * @return the new head of the list or null if the list becomes
- * empy after deletion.
- */
- BlockInfoContiguous listRemove(BlockInfoContiguous head,
- DatanodeStorageInfo storage) {
- if(head == null)
- return null;
- int dnIndex = this.findStorageInfo(storage);
- if(dnIndex < 0) // this block is not on the data-node list
- return head;
-
- BlockInfoContiguous next = this.getNext(dnIndex);
- BlockInfoContiguous prev = this.getPrevious(dnIndex);
- this.setNext(dnIndex, null);
- this.setPrevious(dnIndex, null);
- if(prev != null)
- prev.setNext(prev.findStorageInfo(storage), next);
- if(next != null)
- next.setPrevious(next.findStorageInfo(storage), prev);
- if(this == head) // removing the head
- head = next;
- return head;
- }
-
- /**
- * Remove this block from the list of blocks related to the specified
- * DatanodeDescriptor. Insert it into the head of the list of blocks.
- *
- * @return the new head of the list.
- */
- public BlockInfoContiguous moveBlockToHead(BlockInfoContiguous head,
- DatanodeStorageInfo storage, int curIndex, int headIndex) {
- if (head == this) {
- return this;
- }
- BlockInfoContiguous next = this.setNext(curIndex, head);
- BlockInfoContiguous prev = this.setPrevious(curIndex, null);
-
- head.setPrevious(headIndex, this);
- prev.setNext(prev.findStorageInfo(storage), next);
- if (next != null) {
- next.setPrevious(next.findStorageInfo(storage), prev);
- }
- return this;
- }
-
- /**
- * BlockInfo represents a block that is not being constructed.
- * In order to start modifying the block, the BlockInfo should be converted
- * to {@link BlockInfoContiguousUnderConstruction}.
- * @return {@link BlockUCState#COMPLETE}
- */
- public BlockUCState getBlockUCState() {
- return BlockUCState.COMPLETE;
- }
-
- /**
- * Is this block complete?
- *
- * @return true if the state of the block is {@link BlockUCState#COMPLETE}
- */
- public boolean isComplete() {
- return getBlockUCState().equals(BlockUCState.COMPLETE);
- }
-
- /**
- * Convert a complete block to an under construction block.
- * @return BlockInfoUnderConstruction - an under construction block.
- */
- public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
- BlockUCState s, DatanodeStorageInfo[] targets) {
- if(isComplete()) {
- BlockInfoContiguousUnderConstruction ucBlock =
- new BlockInfoContiguousUnderConstruction(this,
- getBlockCollection().getPreferredBlockReplication(), s, targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
- // the block is already under construction
- BlockInfoContiguousUnderConstruction ucBlock =
- (BlockInfoContiguousUnderConstruction)this;
- ucBlock.setBlockUCState(s);
- ucBlock.setExpectedLocations(targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
-
- @Override
- public int hashCode() {
- // Super implementation is sufficient
- return super.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public LightWeightGSet.LinkedElement getNext() {
- return nextLinkedElement;
- }
-
- @Override
- public void setNext(LightWeightGSet.LinkedElement next) {
- this.nextLinkedElement = next;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 92153ab..58020a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
* Represents a block that is currently being constructed.<br>
* This is usually the last block of a file opened for write or append.
*/
-public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
+public class BlockInfoContiguousUnderConstruction extends BlockInfo {
/** Block state. See {@link BlockUCState} */
private BlockUCState blockUCState;
@@ -182,10 +182,10 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
* the client or it does not have at least a minimal number of replicas
* reported from data-nodes.
*/
- BlockInfoContiguous convertToCompleteBlock() throws IOException {
+ BlockInfo convertToCompleteBlock() throws IOException {
assert getBlockUCState() != BlockUCState.COMPLETE :
"Trying to convert a COMPLETE block";
- return new BlockInfoContiguous(this);
+ return new BlockInfo(this);
}
/** Set expected locations */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 54981fb..45a485b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -543,8 +543,8 @@ public class BlockManager {
int usableReplicas = numReplicas.liveReplicas() +
numReplicas.decommissionedAndDecommissioning();
- if (block instanceof BlockInfoContiguous) {
- BlockCollection bc = ((BlockInfoContiguous) block).getBlockCollection();
+ if (block instanceof BlockInfo) {
+ BlockCollection bc = ((BlockInfo) block).getBlockCollection();
String fileName = (bc == null) ? "[orphaned]" : bc.getName();
out.print(fileName + ": ");
}
@@ -624,7 +624,7 @@ public class BlockManager {
Block commitBlock) throws IOException {
if(commitBlock == null)
return false; // not committing, this is a block allocation retry
- BlockInfoContiguous lastBlock = bc.getLastBlock();
+ BlockInfo lastBlock = bc.getLastBlock();
if(lastBlock == null)
return false; // no blocks in file yet
if(lastBlock.isComplete())
@@ -644,11 +644,11 @@ public class BlockManager {
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- private BlockInfoContiguous completeBlock(final BlockCollection bc,
+ private BlockInfo completeBlock(final BlockCollection bc,
final int blkIndex, boolean force) throws IOException {
if(blkIndex < 0)
return null;
- BlockInfoContiguous curBlock = bc.getBlocks()[blkIndex];
+ BlockInfo curBlock = bc.getBlocks()[blkIndex];
if(curBlock.isComplete())
return curBlock;
BlockInfoContiguousUnderConstruction ucBlock =
@@ -660,7 +660,7 @@ public class BlockManager {
if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED)
throw new IOException(
"Cannot complete block: block has not been COMMITTED by the client");
- BlockInfoContiguous completeBlock = ucBlock.convertToCompleteBlock();
+ BlockInfo completeBlock = ucBlock.convertToCompleteBlock();
// replace penultimate block in file
bc.setBlock(blkIndex, completeBlock);
@@ -678,9 +678,9 @@ public class BlockManager {
return blocksMap.replaceBlock(completeBlock);
}
- private BlockInfoContiguous completeBlock(final BlockCollection bc,
- final BlockInfoContiguous block, boolean force) throws IOException {
- BlockInfoContiguous[] fileBlocks = bc.getBlocks();
+ private BlockInfo completeBlock(final BlockCollection bc,
+ final BlockInfo block, boolean force) throws IOException {
+ BlockInfo[] fileBlocks = bc.getBlocks();
for(int idx = 0; idx < fileBlocks.length; idx++)
if(fileBlocks[idx] == block) {
return completeBlock(bc, idx, force);
@@ -693,7 +693,7 @@ public class BlockManager {
* regardless of whether enough replicas are present. This is necessary
* when tailing edit logs as a Standby.
*/
- public BlockInfoContiguous forceCompleteBlock(final BlockCollection bc,
+ public BlockInfo forceCompleteBlock(final BlockCollection bc,
final BlockInfoContiguousUnderConstruction block) throws IOException {
block.commitBlock(block);
return completeBlock(bc, block, true);
@@ -716,7 +716,7 @@ public class BlockManager {
*/
public LocatedBlock convertLastBlockToUnderConstruction(
BlockCollection bc, long bytesToRemove) throws IOException {
- BlockInfoContiguous oldBlock = bc.getLastBlock();
+ BlockInfo oldBlock = bc.getLastBlock();
if(oldBlock == null ||
bc.getPreferredBlockSize() == oldBlock.getNumBytes() - bytesToRemove)
return null;
@@ -769,7 +769,7 @@ public class BlockManager {
}
private List<LocatedBlock> createLocatedBlockList(
- final BlockInfoContiguous[] blocks,
+ final BlockInfo[] blocks,
final long offset, final long length, final int nrBlocksToReturn,
final AccessMode mode) throws IOException {
int curBlk = 0;
@@ -799,7 +799,7 @@ public class BlockManager {
return results;
}
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous[] blocks,
+ private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
final long endPos, final AccessMode mode) throws IOException {
int curBlk = 0;
long curPos = 0;
@@ -815,7 +815,7 @@ public class BlockManager {
return createLocatedBlock(blocks[curBlk], curPos, mode);
}
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos,
+ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
final AccessMode mode) throws IOException {
final LocatedBlock lb = createLocatedBlock(blk, pos);
if (mode != null) {
@@ -825,7 +825,7 @@ public class BlockManager {
}
/** @return a LocatedBlock for the given block */
- private LocatedBlock createLocatedBlock(final BlockInfoContiguous blk, final long pos
+ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
) throws IOException {
if (blk instanceof BlockInfoContiguousUnderConstruction) {
if (blk.isComplete()) {
@@ -874,7 +874,7 @@ public class BlockManager {
}
/** Create a LocatedBlocks. */
- public LocatedBlocks createLocatedBlocks(final BlockInfoContiguous[] blocks,
+ public LocatedBlocks createLocatedBlocks(final BlockInfo[] blocks,
final long fileSizeExcludeBlocksUnderConstruction,
final boolean isFileUnderConstruction, final long offset,
final long length, final boolean needBlockToken,
@@ -897,7 +897,7 @@ public class BlockManager {
final LocatedBlock lastlb;
final boolean isComplete;
if (!inSnapshot) {
- final BlockInfoContiguous last = blocks[blocks.length - 1];
+ final BlockInfo last = blocks[blocks.length - 1];
final long lastPos = last.isComplete()?
fileSizeExcludeBlocksUnderConstruction - last.getNumBytes()
: fileSizeExcludeBlocksUnderConstruction;
@@ -986,7 +986,7 @@ public class BlockManager {
/**
* Check if a block is replicated to at least the minimum replication.
*/
- public boolean isSufficientlyReplicated(BlockInfoContiguous b) {
+ public boolean isSufficientlyReplicated(BlockInfo b) {
// Compare against the lesser of the minReplication and number of live DNs.
final int replication =
Math.min(minReplication, getDatanodeManager().getNumLiveDataNodes());
@@ -1027,7 +1027,7 @@ public class BlockManager {
if(numBlocks == 0) {
return new BlocksWithLocations(new BlockWithLocations[0]);
}
- Iterator<BlockInfoContiguous> iter = node.getBlockIterator();
+ Iterator<BlockInfo> iter = node.getBlockIterator();
// starting from a random block
int startBlock = ThreadLocalRandom.current().nextInt(numBlocks);
// skip blocks
@@ -1036,7 +1036,7 @@ public class BlockManager {
}
List<BlockWithLocations> results = new ArrayList<BlockWithLocations>();
long totalSize = 0;
- BlockInfoContiguous curBlock;
+ BlockInfo curBlock;
while(totalSize<size && iter.hasNext()) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
@@ -1135,7 +1135,7 @@ public class BlockManager {
public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
final DatanodeInfo dn, String storageID, String reason) throws IOException {
assert namesystem.hasWriteLock();
- final BlockInfoContiguous storedBlock = getStoredBlock(blk.getLocalBlock());
+ final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock());
if (storedBlock == null) {
// Check if the replica is in the blockMap, if not
// ignore the request for now. This could happen when BlockScanner
@@ -1694,7 +1694,7 @@ public class BlockManager {
* Use the blockinfo from the blocksmap to be certain we're working
* with the most up-to-date block information (e.g. genstamp).
*/
- BlockInfoContiguous bi = blocksMap.getStoredBlock(timedOutItems[i]);
+ BlockInfo bi = blocksMap.getStoredBlock(timedOutItems[i]);
if (bi == null) {
continue;
}
@@ -1738,16 +1738,16 @@ public class BlockManager {
*/
private static class BlockToMarkCorrupt {
/** The corrupted block in a datanode. */
- final BlockInfoContiguous corrupted;
+ final BlockInfo corrupted;
/** The corresponding block stored in the BlockManager. */
- final BlockInfoContiguous stored;
+ final BlockInfo stored;
/** The reason to mark corrupt. */
final String reason;
/** The reason code to be stored */
final Reason reasonCode;
- BlockToMarkCorrupt(BlockInfoContiguous corrupted,
- BlockInfoContiguous stored, String reason,
+ BlockToMarkCorrupt(BlockInfo corrupted,
+ BlockInfo stored, String reason,
Reason reasonCode) {
Preconditions.checkNotNull(corrupted, "corrupted is null");
Preconditions.checkNotNull(stored, "stored is null");
@@ -1758,14 +1758,14 @@ public class BlockManager {
this.reasonCode = reasonCode;
}
- BlockToMarkCorrupt(BlockInfoContiguous stored, String reason,
+ BlockToMarkCorrupt(BlockInfo stored, String reason,
Reason reasonCode) {
this(stored, stored, reason, reasonCode);
}
- BlockToMarkCorrupt(BlockInfoContiguous stored, long gs, String reason,
+ BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
Reason reasonCode) {
- this(new BlockInfoContiguous(stored), stored, reason, reasonCode);
+ this(new BlockInfo(stored), stored, reason, reasonCode);
//the corrupted block in datanode has a different generation stamp
corrupted.setGenerationStamp(gs);
}
@@ -1882,10 +1882,10 @@ public class BlockManager {
"longer exists on the DataNode.",
Long.toHexString(context.getReportId()), zombie.getStorageID());
assert(namesystem.hasWriteLock());
- Iterator<BlockInfoContiguous> iter = zombie.getBlockIterator();
+ Iterator<BlockInfo> iter = zombie.getBlockIterator();
int prevBlocks = zombie.numBlocks();
while (iter.hasNext()) {
- BlockInfoContiguous block = iter.next();
+ BlockInfo block = iter.next();
// We assume that a block can be on only one storage in a DataNode.
// That's why we pass in the DatanodeDescriptor rather than the
// DatanodeStorageInfo.
@@ -1943,7 +1943,7 @@ public class BlockManager {
break;
}
- BlockInfoContiguous bi = blocksMap.getStoredBlock(b);
+ BlockInfo bi = blocksMap.getStoredBlock(b);
if (bi == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
@@ -1983,7 +1983,7 @@ public class BlockManager {
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
- Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+ Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
Collection<Block> toRemove = new TreeSet<Block>();
Collection<Block> toInvalidate = new LinkedList<Block>();
Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
@@ -2000,7 +2000,7 @@ public class BlockManager {
removeStoredBlock(b, node);
}
int numBlocksLogged = 0;
- for (BlockInfoContiguous b : toAdd) {
+ for (BlockInfo b : toAdd) {
addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog);
numBlocksLogged++;
}
@@ -2022,7 +2022,7 @@ public class BlockManager {
* Mark block replicas as corrupt except those on the storages in
* newStorages list.
*/
- public void markBlockReplicasAsCorrupt(BlockInfoContiguous block,
+ public void markBlockReplicasAsCorrupt(BlockInfo block,
long oldGenerationStamp, long oldNumBytes,
DatanodeStorageInfo[] newStorages) throws IOException {
assert namesystem.hasWriteLock();
@@ -2091,7 +2091,7 @@ public class BlockManager {
continue;
}
- BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(iblk);
+ BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
// If block does not belong to any file, we are done.
if (storedBlock == null) continue;
@@ -2136,7 +2136,7 @@ public class BlockManager {
private void reportDiff(DatanodeStorageInfo storageInfo,
BlockListAsLongs newReport,
- Collection<BlockInfoContiguous> toAdd, // add to DatanodeDescriptor
+ Collection<BlockInfo> toAdd, // add to DatanodeDescriptor
Collection<Block> toRemove, // remove from DatanodeDescriptor
Collection<Block> toInvalidate, // should be removed from DN
Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
@@ -2144,7 +2144,7 @@ public class BlockManager {
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
- BlockInfoContiguous delimiter = new BlockInfoContiguous(new Block(), (short) 1);
+ BlockInfo delimiter = new BlockInfo(new Block(), (short) 1);
AddBlockResult result = storageInfo.addBlock(delimiter);
assert result == AddBlockResult.ADDED
: "Delimiting block cannot be present in the node";
@@ -2157,7 +2157,7 @@ public class BlockManager {
// scan the report and process newly reported blocks
for (BlockReportReplica iblk : newReport) {
ReplicaState iState = iblk.getState();
- BlockInfoContiguous storedBlock = processReportedBlock(storageInfo,
+ BlockInfo storedBlock = processReportedBlock(storageInfo,
iblk, iState, toAdd, toInvalidate, toCorrupt, toUC);
// move block to the head of the list
@@ -2169,7 +2169,7 @@ public class BlockManager {
// collect blocks that have not been reported
// all of them are next to the delimiter
- Iterator<BlockInfoContiguous> it =
+ Iterator<BlockInfo> it =
storageInfo.new BlockIterator(delimiter.getNext(0));
while(it.hasNext())
toRemove.add(it.next());
@@ -2207,10 +2207,10 @@ public class BlockManager {
* @return the up-to-date stored block, if it should be kept.
* Otherwise, null.
*/
- private BlockInfoContiguous processReportedBlock(
+ private BlockInfo processReportedBlock(
final DatanodeStorageInfo storageInfo,
final Block block, final ReplicaState reportedState,
- final Collection<BlockInfoContiguous> toAdd,
+ final Collection<BlockInfo> toAdd,
final Collection<Block> toInvalidate,
final Collection<BlockToMarkCorrupt> toCorrupt,
final Collection<StatefulBlockInfo> toUC) {
@@ -2231,7 +2231,7 @@ public class BlockManager {
}
// find block by blockId
- BlockInfoContiguous storedBlock = blocksMap.getStoredBlock(block);
+ BlockInfo storedBlock = blocksMap.getStoredBlock(block);
if(storedBlock == null) {
// If blocksMap does not contain reported block id,
// the replica should be removed from the data-node.
@@ -2370,7 +2370,7 @@ public class BlockManager {
*/
private BlockToMarkCorrupt checkReplicaCorrupt(
Block reported, ReplicaState reportedState,
- BlockInfoContiguous storedBlock, BlockUCState ucState,
+ BlockInfo storedBlock, BlockUCState ucState,
DatanodeDescriptor dn) {
switch(reportedState) {
case FINALIZED:
@@ -2443,7 +2443,7 @@ public class BlockManager {
}
}
- private boolean isBlockUnderConstruction(BlockInfoContiguous storedBlock,
+ private boolean isBlockUnderConstruction(BlockInfo storedBlock,
BlockUCState ucState, ReplicaState reportedState) {
switch(reportedState) {
case FINALIZED:
@@ -2487,7 +2487,7 @@ public class BlockManager {
*
* @throws IOException
*/
- private void addStoredBlockImmediate(BlockInfoContiguous storedBlock,
+ private void addStoredBlockImmediate(BlockInfo storedBlock,
DatanodeStorageInfo storageInfo)
throws IOException {
assert (storedBlock != null && namesystem.hasWriteLock());
@@ -2519,13 +2519,13 @@ public class BlockManager {
* needed replications if this takes care of the problem.
* @return the block that is stored in blockMap.
*/
- private Block addStoredBlock(final BlockInfoContiguous block,
+ private Block addStoredBlock(final BlockInfo block,
DatanodeStorageInfo storageInfo,
DatanodeDescriptor delNodeHint,
boolean logEveryBlock)
throws IOException {
assert block != null && namesystem.hasWriteLock();
- BlockInfoContiguous storedBlock;
+ BlockInfo storedBlock;
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
if (block instanceof BlockInfoContiguousUnderConstruction) {
//refresh our copy in case the block got completed in another thread
@@ -2623,7 +2623,7 @@ public class BlockManager {
return storedBlock;
}
- private void logAddStoredBlock(BlockInfoContiguous storedBlock,
+ private void logAddStoredBlock(BlockInfo storedBlock,
DatanodeDescriptor node) {
if (!blockLog.isInfoEnabled()) {
return;
@@ -2651,7 +2651,7 @@ public class BlockManager {
*
* @param blk Block whose corrupt replicas need to be invalidated
*/
- private void invalidateCorruptReplicas(BlockInfoContiguous blk) {
+ private void invalidateCorruptReplicas(BlockInfo blk) {
Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk);
boolean removedFromBlocksMap = true;
if (nodes == null)
@@ -2730,7 +2730,7 @@ public class BlockManager {
long nrInvalid = 0, nrOverReplicated = 0;
long nrUnderReplicated = 0, nrPostponed = 0, nrUnderConstruction = 0;
long startTimeMisReplicatedScan = Time.monotonicNow();
- Iterator<BlockInfoContiguous> blocksItr = blocksMap.getBlocks().iterator();
+ Iterator<BlockInfo> blocksItr = blocksMap.getBlocks().iterator();
long totalBlocks = blocksMap.size();
replicationQueuesInitProgress = 0;
long totalProcessed = 0;
@@ -2742,7 +2742,7 @@ public class BlockManager {
namesystem.writeLockInterruptibly();
try {
while (processed < numBlocksPerIteration && blocksItr.hasNext()) {
- BlockInfoContiguous block = blocksItr.next();
+ BlockInfo block = blocksItr.next();
MisReplicationResult res = processMisReplicatedBlock(block);
if (LOG.isTraceEnabled()) {
LOG.trace("block " + block + ": " + res);
@@ -2817,7 +2817,7 @@ public class BlockManager {
* appropriate queues if necessary, and returns a result code indicating
* what happened with it.
*/
- private MisReplicationResult processMisReplicatedBlock(BlockInfoContiguous block) {
+ private MisReplicationResult processMisReplicatedBlock(BlockInfo block) {
if (block.isDeleted()) {
// block does not belong to any file
addToInvalidates(block);
@@ -3157,7 +3157,7 @@ public class BlockManager {
ReplicaState reportedState, DatanodeDescriptor delHintNode)
throws IOException {
// blockReceived reports a finalized block
- Collection<BlockInfoContiguous> toAdd = new LinkedList<BlockInfoContiguous>();
+ Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
Collection<Block> toInvalidate = new LinkedList<Block>();
Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
@@ -3174,7 +3174,7 @@ public class BlockManager {
addStoredBlockUnderConstruction(b, storageInfo);
}
long numBlocksLogged = 0;
- for (BlockInfoContiguous b : toAdd) {
+ for (BlockInfo b : toAdd) {
addStoredBlock(b, storageInfo, delHintNode, numBlocksLogged < maxNumBlocksToLog);
numBlocksLogged++;
}
@@ -3301,7 +3301,7 @@ public class BlockManager {
* @param b - the block being tested
* @return count of live nodes for this block
*/
- int countLiveNodes(BlockInfoContiguous b) {
+ int countLiveNodes(BlockInfo b) {
if (!namesystem.isInStartupSafeMode()) {
return countNodes(b).liveReplicas();
}
@@ -3380,7 +3380,7 @@ public class BlockManager {
return blocksMap.size();
}
- public DatanodeStorageInfo[] getStorages(BlockInfoContiguous block) {
+ public DatanodeStorageInfo[] getStorages(BlockInfo block) {
final DatanodeStorageInfo[] storages = new DatanodeStorageInfo[block.numNodes()];
int i = 0;
for(DatanodeStorageInfo s : blocksMap.getStorages(block)) {
@@ -3409,7 +3409,7 @@ public class BlockManager {
}
}
- public BlockInfoContiguous getStoredBlock(Block block) {
+ public BlockInfo getStoredBlock(Block block) {
return blocksMap.getStoredBlock(block);
}
@@ -3462,8 +3462,8 @@ public class BlockManager {
* replicated.
*/
public boolean checkBlocksProperlyReplicated(
- String src, BlockInfoContiguous[] blocks) {
- for (BlockInfoContiguous b: blocks) {
+ String src, BlockInfo[] blocks) {
+ for (BlockInfo b: blocks) {
if (!b.isComplete()) {
final BlockInfoContiguousUnderConstruction uc =
(BlockInfoContiguousUnderConstruction)b;
@@ -3578,7 +3578,7 @@ public class BlockManager {
return this.neededReplications.getCorruptReplOneBlockSize();
}
- public BlockInfoContiguous addBlockCollection(BlockInfoContiguous block,
+ public BlockInfo addBlockCollection(BlockInfo block,
BlockCollection bc) {
return blocksMap.addBlockCollection(block, bc);
}
@@ -3818,7 +3818,7 @@ public class BlockManager {
/**
* A simple result enum for the result of
- * {@link BlockManager#processMisReplicatedBlock(BlockInfoContiguous)}.
+ * {@link BlockManager#processMisReplicatedBlock(BlockInfo)}.
*/
enum MisReplicationResult {
/** The block should be invalidated since it belongs to a deleted file. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 5e7d34f..d06d503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -36,10 +36,10 @@ import com.google.common.collect.Iterables;
*/
class BlocksMap {
private static class StorageIterator implements Iterator<DatanodeStorageInfo> {
- private final BlockInfoContiguous blockInfo;
+ private final BlockInfo blockInfo;
private int nextIdx = 0;
- StorageIterator(BlockInfoContiguous blkInfo) {
+ StorageIterator(BlockInfo blkInfo) {
this.blockInfo = blkInfo;
}
@@ -63,14 +63,14 @@ class BlocksMap {
/** Constant {@link LightWeightGSet} capacity. */
private final int capacity;
- private GSet<Block, BlockInfoContiguous> blocks;
+ private GSet<Block, BlockInfo> blocks;
BlocksMap(int capacity) {
// Use 2% of total memory to size the GSet capacity
this.capacity = capacity;
- this.blocks = new LightWeightGSet<Block, BlockInfoContiguous>(capacity) {
+ this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity) {
@Override
- public Iterator<BlockInfoContiguous> iterator() {
+ public Iterator<BlockInfo> iterator() {
SetIterator iterator = new SetIterator();
/*
* Not tracking any modifications to set. As this set will be used
@@ -97,15 +97,15 @@ class BlocksMap {
}
BlockCollection getBlockCollection(Block b) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
return (info != null) ? info.getBlockCollection() : null;
}
/**
* Add block b belonging to the specified block collection to the map.
*/
- BlockInfoContiguous addBlockCollection(BlockInfoContiguous b, BlockCollection bc) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo addBlockCollection(BlockInfo b, BlockCollection bc) {
+ BlockInfo info = blocks.get(b);
if (info != b) {
info = b;
blocks.put(info);
@@ -120,7 +120,7 @@ class BlocksMap {
* and remove all data-node locations associated with the block.
*/
void removeBlock(Block block) {
- BlockInfoContiguous blockInfo = blocks.remove(block);
+ BlockInfo blockInfo = blocks.remove(block);
if (blockInfo == null)
return;
@@ -132,7 +132,7 @@ class BlocksMap {
}
/** Returns the block object it it exists in the map. */
- BlockInfoContiguous getStoredBlock(Block b) {
+ BlockInfo getStoredBlock(Block b) {
return blocks.get(b);
}
@@ -164,7 +164,7 @@ class BlocksMap {
* For a block that has already been retrieved from the BlocksMap
* returns {@link Iterable} of the storages the block belongs to.
*/
- Iterable<DatanodeStorageInfo> getStorages(final BlockInfoContiguous storedBlock) {
+ Iterable<DatanodeStorageInfo> getStorages(final BlockInfo storedBlock) {
return new Iterable<DatanodeStorageInfo>() {
@Override
public Iterator<DatanodeStorageInfo> iterator() {
@@ -175,7 +175,7 @@ class BlocksMap {
/** counts number of containing nodes. Better than using iterator. */
int numNodes(Block b) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
return info == null ? 0 : info.numNodes();
}
@@ -185,7 +185,7 @@ class BlocksMap {
* only if it does not belong to any file and data-nodes.
*/
boolean removeNode(Block b, DatanodeDescriptor node) {
- BlockInfoContiguous info = blocks.get(b);
+ BlockInfo info = blocks.get(b);
if (info == null)
return false;
@@ -203,7 +203,7 @@ class BlocksMap {
return blocks.size();
}
- Iterable<BlockInfoContiguous> getBlocks() {
+ Iterable<BlockInfo> getBlocks() {
return blocks;
}
@@ -218,8 +218,8 @@ class BlocksMap {
* @param newBlock - block for replacement
* @return new block
*/
- BlockInfoContiguous replaceBlock(BlockInfoContiguous newBlock) {
- BlockInfoContiguous currentBlock = blocks.get(newBlock);
+ BlockInfo replaceBlock(BlockInfo newBlock) {
+ BlockInfo currentBlock = blocks.get(newBlock);
assert currentBlock != null : "the block if not in blocksMap";
// replace block in data-node lists
for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index bf5ece9..a0f3503 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -369,7 +369,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
* @param file The file.
*/
private void rescanFile(CacheDirective directive, INodeFile file) {
- BlockInfoContiguous[] blockInfos = file.getBlocks();
+ BlockInfo[] blockInfos = file.getBlocks();
// Increment the "needed" statistics
directive.addFilesNeeded(1);
@@ -394,7 +394,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
}
long cachedTotal = 0;
- for (BlockInfoContiguous blockInfo : blockInfos) {
+ for (BlockInfo blockInfo : blockInfos) {
if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) {
// We don't try to cache blocks that are under construction.
LOG.trace("Directive {}: can't cache block {} because it is in state "
@@ -453,7 +453,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
}
private String findReasonForNotCaching(CachedBlock cblock,
- BlockInfoContiguous blockInfo) {
+ BlockInfo blockInfo) {
if (blockInfo == null) {
// Somehow, a cache report with the block arrived, but the block
// reports from the DataNode haven't (yet?) described such a block.
@@ -513,7 +513,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
iter.remove();
}
}
- BlockInfoContiguous blockInfo = blockManager.
+ BlockInfo blockInfo = blockManager.
getStoredBlock(new Block(cblock.getBlockId()));
String reason = findReasonForNotCaching(cblock, blockInfo);
int neededCached = 0;
@@ -628,7 +628,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
List<DatanodeDescriptor> pendingCached) {
// To figure out which replicas can be cached, we consult the
// blocksMap. We don't want to try to cache a corrupt replica, though.
- BlockInfoContiguous blockInfo = blockManager.
+ BlockInfo blockInfo = blockManager.
getStoredBlock(new Block(cachedBlock.getBlockId()));
if (blockInfo == null) {
LOG.debug("Block {}: can't add new cached replicas," +
@@ -667,7 +667,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
Iterator<CachedBlock> it = datanode.getPendingCached().iterator();
while (it.hasNext()) {
CachedBlock cBlock = it.next();
- BlockInfoContiguous info =
+ BlockInfo info =
blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
if (info != null) {
pendingBytes -= info.getNumBytes();
@@ -677,7 +677,7 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
// Add pending uncached blocks from effective capacity
while (it.hasNext()) {
CachedBlock cBlock = it.next();
- BlockInfoContiguous info =
+ BlockInfo info =
blockManager.getStoredBlock(new Block(cBlock.getBlockId()));
if (info != null) {
pendingBytes += info.getNumBytes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 4731ad4..02abc1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -335,7 +335,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
*/
- boolean removeBlock(BlockInfoContiguous b) {
+ boolean removeBlock(BlockInfo b) {
final DatanodeStorageInfo s = b.findStorageInfo(this);
// if block exists on this datanode
if (s != null) {
@@ -348,7 +348,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
* Remove block from the list of blocks belonging to the data-node. Remove
* data-node from the block.
*/
- boolean removeBlock(String storageID, BlockInfoContiguous b) {
+ boolean removeBlock(String storageID, BlockInfo b) {
DatanodeStorageInfo s = getStorageInfo(storageID);
if (s != null) {
return s.removeBlock(b);
@@ -537,12 +537,12 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
- private static class BlockIterator implements Iterator<BlockInfoContiguous> {
+ private static class BlockIterator implements Iterator<BlockInfo> {
private int index = 0;
- private final List<Iterator<BlockInfoContiguous>> iterators;
+ private final List<Iterator<BlockInfo>> iterators;
private BlockIterator(final DatanodeStorageInfo... storages) {
- List<Iterator<BlockInfoContiguous>> iterators = new ArrayList<Iterator<BlockInfoContiguous>>();
+ List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
for (DatanodeStorageInfo e : storages) {
iterators.add(e.getBlockIterator());
}
@@ -556,7 +556,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
@Override
- public BlockInfoContiguous next() {
+ public BlockInfo next() {
update();
return iterators.get(index).next();
}
@@ -573,10 +573,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
- Iterator<BlockInfoContiguous> getBlockIterator() {
+ Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(getStorageInfos());
}
- Iterator<BlockInfoContiguous> getBlockIterator(final String storageID) {
+ Iterator<BlockInfo> getBlockIterator(final String storageID) {
return new BlockIterator(getStorageInfo(storageID));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index c6c9001..57e81b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -25,9 +25,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
@@ -83,10 +80,10 @@ public class DatanodeStorageInfo {
/**
* Iterates over the list of blocks belonging to the data-node.
*/
- class BlockIterator implements Iterator<BlockInfoContiguous> {
- private BlockInfoContiguous current;
+ class BlockIterator implements Iterator<BlockInfo> {
+ private BlockInfo current;
- BlockIterator(BlockInfoContiguous head) {
+ BlockIterator(BlockInfo head) {
this.current = head;
}
@@ -94,8 +91,8 @@ public class DatanodeStorageInfo {
return current != null;
}
- public BlockInfoContiguous next() {
- BlockInfoContiguous res = current;
+ public BlockInfo next() {
+ BlockInfo res = current;
current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this));
return res;
}
@@ -115,7 +112,7 @@ public class DatanodeStorageInfo {
private volatile long remaining;
private long blockPoolUsed;
- private volatile BlockInfoContiguous blockList = null;
+ private volatile BlockInfo blockList = null;
private int numBlocks = 0;
// The ID of the last full block report which updated this storage.
@@ -229,7 +226,7 @@ public class DatanodeStorageInfo {
return blockPoolUsed;
}
- public AddBlockResult addBlock(BlockInfoContiguous b) {
+ public AddBlockResult addBlock(BlockInfo b) {
// First check whether the block belongs to a different storage
// on the same DN.
AddBlockResult result = AddBlockResult.ADDED;
@@ -254,7 +251,7 @@ public class DatanodeStorageInfo {
return result;
}
- public boolean removeBlock(BlockInfoContiguous b) {
+ public boolean removeBlock(BlockInfo b) {
blockList = b.listRemove(blockList, this);
if (b.removeStorage(this)) {
numBlocks--;
@@ -268,7 +265,7 @@ public class DatanodeStorageInfo {
return numBlocks;
}
- Iterator<BlockInfoContiguous> getBlockIterator() {
+ Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(blockList);
}
@@ -277,7 +274,7 @@ public class DatanodeStorageInfo {
* Move block to the head of the list of blocks belonging to the data-node.
* @return the index of the head of the blockList
*/
- int moveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex) {
+ int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) {
blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex);
return curIndex;
}
@@ -287,7 +284,7 @@ public class DatanodeStorageInfo {
* @return the head of the blockList
*/
@VisibleForTesting
- BlockInfoContiguous getBlockListHeadForTesting(){
+ BlockInfo getBlockListHeadForTesting(){
return blockList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 5f7366e..639d2b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -100,7 +100,7 @@ public class DecommissionManager {
* reports or other events. Before being finally marking as decommissioned,
* another check is done with the actual block map.
*/
- private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+ private final TreeMap<DatanodeDescriptor, AbstractList<BlockInfo>>
decomNodeBlocks;
/**
@@ -248,7 +248,7 @@ public class DecommissionManager {
* Full-strength replication is not always necessary, hence "sufficient".
* @return true if sufficient, else false.
*/
- private boolean isSufficientlyReplicated(BlockInfoContiguous block,
+ private boolean isSufficientlyReplicated(BlockInfo block,
BlockCollection bc,
NumberReplicas numberReplicas) {
final int numExpected = bc.getPreferredBlockReplication();
@@ -412,7 +412,7 @@ public class DecommissionManager {
}
private void check() {
- final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>>
+ final Iterator<Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>>
it = new CyclicIteration<>(decomNodeBlocks, iterkey).iterator();
final LinkedList<DatanodeDescriptor> toRemove = new LinkedList<>();
@@ -420,10 +420,10 @@ public class DecommissionManager {
&& !exceededNumBlocksPerCheck()
&& !exceededNumNodesPerCheck()) {
numNodesChecked++;
- final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfoContiguous>>
+ final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>
entry = it.next();
final DatanodeDescriptor dn = entry.getKey();
- AbstractList<BlockInfoContiguous> blocks = entry.getValue();
+ AbstractList<BlockInfo> blocks = entry.getValue();
boolean fullScan = false;
if (blocks == null) {
// This is a newly added datanode, run through its list to schedule
@@ -495,7 +495,7 @@ public class DecommissionManager {
* datanode.
*/
private void pruneSufficientlyReplicated(final DatanodeDescriptor datanode,
- AbstractList<BlockInfoContiguous> blocks) {
+ AbstractList<BlockInfo> blocks) {
processBlocksForDecomInternal(datanode, blocks.iterator(), null, true);
}
@@ -509,9 +509,9 @@ public class DecommissionManager {
* @param datanode
* @return List of insufficiently replicated blocks
*/
- private AbstractList<BlockInfoContiguous> handleInsufficientlyReplicated(
+ private AbstractList<BlockInfo> handleInsufficientlyReplicated(
final DatanodeDescriptor datanode) {
- AbstractList<BlockInfoContiguous> insufficient = new ChunkedArrayList<>();
+ AbstractList<BlockInfo> insufficient = new ChunkedArrayList<>();
processBlocksForDecomInternal(datanode, datanode.getBlockIterator(),
insufficient, false);
return insufficient;
@@ -535,8 +535,8 @@ public class DecommissionManager {
*/
private void processBlocksForDecomInternal(
final DatanodeDescriptor datanode,
- final Iterator<BlockInfoContiguous> it,
- final List<BlockInfoContiguous> insufficientlyReplicated,
+ final Iterator<BlockInfo> it,
+ final List<BlockInfo> insufficientlyReplicated,
boolean pruneSufficientlyReplicated) {
boolean firstReplicationLog = true;
int underReplicatedBlocks = 0;
@@ -544,7 +544,7 @@ public class DecommissionManager {
int underReplicatedInOpenFiles = 0;
while (it.hasNext()) {
numBlocksChecked++;
- final BlockInfoContiguous block = it.next();
+ final BlockInfo block = it.next();
// Remove the block from the list if it's no longer in the block map,
// e.g. the containing file has been deleted
if (blockManager.blocksMap.getStoredBlock(block) == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 307bd59..3b5f973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -235,7 +235,7 @@ class FSDirWriteFileOp {
return onRetryBlock[0];
} else {
// add new chosen targets to already allocated block and return
- BlockInfoContiguous lastBlockInFile = pendingFile.getLastBlock();
+ BlockInfo lastBlockInFile = pendingFile.getLastBlock();
((BlockInfoContiguousUnderConstruction) lastBlockInFile)
.setExpectedLocations(targets);
offset = pendingFile.computeFileSize();
@@ -507,7 +507,7 @@ class FSDirWriteFileOp {
/**
* Add a block to the file. Returns a reference to the added block.
*/
- private static BlockInfoContiguous addBlock(
+ private static BlockInfo addBlock(
FSDirectory fsd, String path, INodesInPath inodesInPath, Block block,
DatanodeStorageInfo[] targets) throws IOException {
fsd.writeLock();
@@ -608,7 +608,7 @@ class FSDirWriteFileOp {
}
}
final INodeFile file = fsn.checkLease(src, clientName, inode, fileId);
- BlockInfoContiguous lastBlockInFile = file.getLastBlock();
+ BlockInfo lastBlockInFile = file.getLastBlock();
if (!Block.matchingIdAndGenStamp(previousBlock, lastBlockInFile)) {
// The block that the client claims is the current last block
// doesn't match up with what we think is the last block. There are
@@ -636,7 +636,7 @@ class FSDirWriteFileOp {
// changed the namesystem state yet.
// We run this analysis again in Part II where case 4 is impossible.
- BlockInfoContiguous penultimateBlock = file.getPenultimateBlock();
+ BlockInfo penultimateBlock = file.getPenultimateBlock();
if (previous == null &&
lastBlockInFile != null &&
lastBlockInFile.getNumBytes() >= file.getPreferredBlockSize() &&
@@ -762,7 +762,7 @@ class FSDirWriteFileOp {
long id, PermissionStatus permissions, long mtime, long atime,
short replication, long preferredBlockSize, byte storagePolicyId) {
return new INodeFile(id, null, permissions, mtime, atime,
- BlockInfoContiguous.EMPTY_ARRAY, replication, preferredBlockSize,
+ BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
storagePolicyId);
}
@@ -802,7 +802,7 @@ class FSDirWriteFileOp {
DatanodeStorageInfo[] targets)
throws IOException {
assert fsn.hasWriteLock();
- BlockInfoContiguous b = addBlock(fsn.dir, src, inodesInPath, newBlock,
+ BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock,
targets);
NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 8fdd2d7..5b8650e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -33,10 +33,8 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
-import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
@@ -51,7 +49,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -919,7 +917,7 @@ public class FSDirectory implements Closeable {
unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
if(! onBlockBoundary) {
- BlockInfoContiguous oldBlock = file.getLastBlock();
+ BlockInfo oldBlock = file.getLastBlock();
Block tBlk =
getFSNamesystem().prepareFileForTruncate(iip,
clientName, clientMachine, file.computeFileSize() - newLength,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b88df72/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 83e52bc..dec9709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.Storage.FormatConfirmable;
@@ -773,10 +773,10 @@ public class FSEditLog implements LogsPurgeable {
public void logAddBlock(String path, INodeFile file) {
Preconditions.checkArgument(file.isUnderConstruction());
- BlockInfoContiguous[] blocks = file.getBlocks();
+ BlockInfo[] blocks = file.getBlocks();
Preconditions.checkState(blocks != null && blocks.length > 0);
- BlockInfoContiguous pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
- BlockInfoContiguous lastBlock = blocks[blocks.length - 1];
+ BlockInfo pBlock = blocks.length > 1 ? blocks[blocks.length - 2] : null;
+ BlockInfo lastBlock = blocks[blocks.length - 1];
AddBlockOp op = AddBlockOp.getInstance(cache.get()).setPath(path)
.setPenultimateBlock(pBlock).setLastBlock(lastBlock);
logEdit(op);